doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/conv2d.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// Conv2D Kernel (NCHW, f16)
|
|
4
4
|
|
|
5
5
|
enable f16;
|
|
@@ -19,13 +19,6 @@ export interface DequantOptions extends OutputBufferOptions, OutputOffsetOptions
|
|
|
19
19
|
dequantTileShape?: 'vec4' | 'scalar';
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
-
/** Context for dequant variant selection rules. */
|
|
23
|
-
export interface DequantSelectionContext {
|
|
24
|
-
hasSubgroups: boolean;
|
|
25
|
-
wantsF16Out: boolean;
|
|
26
|
-
useVec4: boolean;
|
|
27
|
-
}
|
|
28
|
-
|
|
29
22
|
/**
|
|
30
23
|
* Select the best dequantization kernel variant
|
|
31
24
|
*/
|
|
@@ -88,15 +81,6 @@ export declare function dequantizeQ6K(
|
|
|
88
81
|
options?: DequantOptions
|
|
89
82
|
): Promise<Tensor>;
|
|
90
83
|
|
|
91
|
-
/**
|
|
92
|
-
* Run Q8_0 dequantization
|
|
93
|
-
*/
|
|
94
|
-
export declare function dequantizeQ8_0(
|
|
95
|
-
quantized: GPUBuffer,
|
|
96
|
-
numBlocks: number,
|
|
97
|
-
options?: DequantOptions
|
|
98
|
-
): Promise<Tensor>;
|
|
99
|
-
|
|
100
84
|
/**
|
|
101
85
|
* Record Q4_K_M dequantization (batched, no submit)
|
|
102
86
|
*/
|
|
@@ -4,7 +4,7 @@ import { getDevice, getKernelCapabilities } from '../device.js';
|
|
|
4
4
|
import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
|
|
5
5
|
import { createTensor } from '../tensor.js';
|
|
6
6
|
import { GPU_LIMITS, TILE_SIZES, WORKGROUP_SIZES, DEQUANT_DISPATCH } from './constants.js';
|
|
7
|
-
import { Q6K_BLOCK_BYTES
|
|
7
|
+
import { Q6K_BLOCK_BYTES } from '../../loader/quantization-constants.js';
|
|
8
8
|
import { dispatch, recordDispatch } from './dispatch.js';
|
|
9
9
|
import { getPipelineFast, createUniformBufferWithView, getOrCreateBindGroupLayout, getKernelConfig } from './utils.js';
|
|
10
10
|
import { releaseUniformBuffer } from '../uniform-cache.js';
|
|
@@ -439,76 +439,6 @@ export async function dequantizeQ6K(
|
|
|
439
439
|
}
|
|
440
440
|
|
|
441
441
|
|
|
442
|
-
export async function dequantizeQ8_0(
|
|
443
|
-
quantized,
|
|
444
|
-
numBlocks,
|
|
445
|
-
options = {}
|
|
446
|
-
) {
|
|
447
|
-
const device = getDevice();
|
|
448
|
-
const {
|
|
449
|
-
outputOffset = 0,
|
|
450
|
-
outputBuffer = null,
|
|
451
|
-
outputDtype = 'f16', // Q8_0 outputs f16 for now
|
|
452
|
-
} = options;
|
|
453
|
-
|
|
454
|
-
// Q8_0 only has f16 output kernel currently
|
|
455
|
-
const pipeline = await getPipelineFast('dequant', 'q8_0_f16out');
|
|
456
|
-
|
|
457
|
-
// Q8_0: 32 elements per block
|
|
458
|
-
const bytesPerElem = outputDtype === 'f16' ? 2 : 4;
|
|
459
|
-
const outputSize = numBlocks * Q8_0_BLOCK_SIZE * bytesPerElem;
|
|
460
|
-
|
|
461
|
-
// Create output buffer if not provided
|
|
462
|
-
const ownedOutput = outputBuffer ? null : acquireBuffer(outputSize, undefined, 'q8_0_dequant_output');
|
|
463
|
-
const output = outputBuffer || ownedOutput;
|
|
464
|
-
|
|
465
|
-
// Calculate workgroups for 2D dispatch
|
|
466
|
-
const maxWorkgroups = GPU_LIMITS.MAX_WORKGROUPS;
|
|
467
|
-
const workgroupsX = Math.min(numBlocks, maxWorkgroups);
|
|
468
|
-
|
|
469
|
-
// Create uniform buffer
|
|
470
|
-
const uniformBuffer = createUniformBufferWithView(
|
|
471
|
-
'q8_0_dequant_uniforms',
|
|
472
|
-
16,
|
|
473
|
-
(view) => {
|
|
474
|
-
view.setUint32(0, numBlocks, true);
|
|
475
|
-
view.setUint32(4, outputOffset, true);
|
|
476
|
-
view.setUint32(8, workgroupsX, true); // workgroups_x for 2D dispatch
|
|
477
|
-
view.setUint32(12, 0, true); // padding
|
|
478
|
-
},
|
|
479
|
-
null,
|
|
480
|
-
device
|
|
481
|
-
);
|
|
482
|
-
|
|
483
|
-
try {
|
|
484
|
-
const bindGroup = device.createBindGroup({
|
|
485
|
-
label: 'q8_0_dequant_bind_group',
|
|
486
|
-
layout: pipeline.getBindGroupLayout(0),
|
|
487
|
-
entries: [
|
|
488
|
-
{ binding: 0, resource: { buffer: uniformBuffer } },
|
|
489
|
-
{ binding: 1, resource: { buffer: quantized } },
|
|
490
|
-
{ binding: 2, resource: { buffer: output } },
|
|
491
|
-
],
|
|
492
|
-
});
|
|
493
|
-
|
|
494
|
-
const workgroups = [
|
|
495
|
-
workgroupsX,
|
|
496
|
-
numBlocks > maxWorkgroups ? Math.ceil(numBlocks / maxWorkgroups) : 1,
|
|
497
|
-
1
|
|
498
|
-
];
|
|
499
|
-
|
|
500
|
-
dispatch(device, pipeline, bindGroup, workgroups, 'q8_0_dequant');
|
|
501
|
-
} catch (error) {
|
|
502
|
-
cleanupDequantResources(uniformBuffer, [ownedOutput]);
|
|
503
|
-
throw error;
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
releaseUniformBuffer(uniformBuffer);
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
const dtype = selectSharedRuleValue('shared', 'dtype', 'f16OrF32FromDtype', { dtype: outputDtype });
|
|
510
|
-
return createTensor(output, dtype, [numBlocks * Q8_0_BLOCK_SIZE], 'q8_0_dequant_output');
|
|
511
|
-
}
|
|
512
442
|
|
|
513
443
|
|
|
514
444
|
export async function recordDequantize(
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/dequant_mxfp4_expert.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// MXFP4 Dequantization Kernel (expert slice, f16 output)
|
|
4
4
|
|
|
5
5
|
enable f16;
|
|
@@ -12,11 +12,9 @@
|
|
|
12
12
|
// Q4_K constants
|
|
13
13
|
const QK_K: u32 = 256u;
|
|
14
14
|
const SUBBLOCK_SIZE: u32 = 32u;
|
|
15
|
-
const NUM_SUBBLOCKS: u32 = 8u;
|
|
16
15
|
|
|
17
|
-
// Tunable workgroup
|
|
16
|
+
// Tunable workgroup size
|
|
18
17
|
override WORKGROUP_SIZE_MAIN: u32 = 256u;
|
|
19
|
-
override WORKGROUP_SIZE_F16: u32 = 256u;
|
|
20
18
|
|
|
21
19
|
struct Uniforms {
|
|
22
20
|
num_blocks: u32,
|
|
@@ -146,55 +144,3 @@ fn main(
|
|
|
146
144
|
output[out_idx] = dequant;
|
|
147
145
|
}
|
|
148
146
|
|
|
149
|
-
// FP16 output variant for when downstream consumers want f16
|
|
150
|
-
@compute @workgroup_size(WORKGROUP_SIZE_F16, 1, 1)
|
|
151
|
-
fn main_f16_out(
|
|
152
|
-
@builtin(local_invocation_id) local_id: vec3<u32>,
|
|
153
|
-
@builtin(workgroup_id) workgroup_id: vec3<u32>
|
|
154
|
-
) {
|
|
155
|
-
let block_idx = workgroup_id.x;
|
|
156
|
-
let elem_idx = local_id.x;
|
|
157
|
-
|
|
158
|
-
if (block_idx >= u.num_blocks) {
|
|
159
|
-
return;
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
let block = quantized[block_idx];
|
|
163
|
-
let scale_word0 = block.scales[0];
|
|
164
|
-
let scale_word1 = block.scales[1];
|
|
165
|
-
let scale_word2 = block.scales[2];
|
|
166
|
-
|
|
167
|
-
if (elem_idx == 0u) {
|
|
168
|
-
shared_d = unpack_f16_lo(block.d);
|
|
169
|
-
shared_dmin = unpack_f16_hi(block.d);
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
// Threads 0-7 load scales and mins
|
|
173
|
-
if (elem_idx < 8u) {
|
|
174
|
-
let sm = get_scale_min_k4(scale_word0, scale_word1, scale_word2, elem_idx);
|
|
175
|
-
shared_scales[elem_idx] = f32(sm.x);
|
|
176
|
-
shared_mins[elem_idx] = f32(sm.y);
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
workgroupBarrier();
|
|
180
|
-
|
|
181
|
-
let subblock_idx = elem_idx / SUBBLOCK_SIZE; // 0-7
|
|
182
|
-
let scale = shared_d * shared_scales[subblock_idx];
|
|
183
|
-
let min_val = shared_dmin * shared_mins[subblock_idx];
|
|
184
|
-
|
|
185
|
-
let chunk = elem_idx / 64u;
|
|
186
|
-
let pos_in_chunk = elem_idx % 64u;
|
|
187
|
-
let use_upper = pos_in_chunk >= 32u;
|
|
188
|
-
let byte_in_range = select(pos_in_chunk, pos_in_chunk - 32u, use_upper);
|
|
189
|
-
let byte_idx = chunk * 32u + byte_in_range;
|
|
190
|
-
let word_idx = byte_idx / 4u;
|
|
191
|
-
let byte_in_word = byte_idx % 4u;
|
|
192
|
-
let byte_val = (block.qs[word_idx] >> (byte_in_word * 8u)) & 0xFFu;
|
|
193
|
-
let q = select(byte_val & 0xFu, (byte_val >> 4u) & 0xFu, use_upper);
|
|
194
|
-
// llama.cpp formula: dequant = d * scale * q - dmin * min
|
|
195
|
-
let dequant = scale * f32(q) - min_val;
|
|
196
|
-
|
|
197
|
-
// Note: This writes f32, but caller can cast if needed
|
|
198
|
-
let out_idx = u.output_offset + block_idx * QK_K + elem_idx;
|
|
199
|
-
output[out_idx] = dequant;
|
|
200
|
-
}
|
|
@@ -21,6 +21,18 @@ export declare function dispatch(
|
|
|
21
21
|
label?: string
|
|
22
22
|
): void;
|
|
23
23
|
|
|
24
|
+
/**
|
|
25
|
+
* Polymorphic dispatch that picks between submit-and-dispatch (GPUDevice)
|
|
26
|
+
* and record-only (CommandRecorder) based on the target's shape.
|
|
27
|
+
*/
|
|
28
|
+
export declare function dispatchKernel(
|
|
29
|
+
target: GPUDevice | CommandRecorder | null,
|
|
30
|
+
pipeline: GPUComputePipeline,
|
|
31
|
+
bindGroup: GPUBindGroup,
|
|
32
|
+
workgroups: number | [number, number, number],
|
|
33
|
+
label?: string
|
|
34
|
+
): void;
|
|
35
|
+
|
|
24
36
|
/**
|
|
25
37
|
* Record a compute pass to a CommandRecorder (no submit)
|
|
26
38
|
* Use for kernels in the batched pipeline path
|
|
@@ -58,100 +70,3 @@ export declare function recordDispatchIndirect(
|
|
|
58
70
|
label?: string
|
|
59
71
|
): void;
|
|
60
72
|
|
|
61
|
-
/**
|
|
62
|
-
* Dispatch with multiple bind groups
|
|
63
|
-
* For kernels that use multiple bind group sets
|
|
64
|
-
*/
|
|
65
|
-
export declare function dispatchMultiBindGroup(
|
|
66
|
-
device: GPUDevice,
|
|
67
|
-
pipeline: GPUComputePipeline,
|
|
68
|
-
bindGroups: GPUBindGroup[],
|
|
69
|
-
workgroups: number | [number, number, number],
|
|
70
|
-
label?: string
|
|
71
|
-
): void;
|
|
72
|
-
|
|
73
|
-
/**
|
|
74
|
-
* Calculate workgroup count for 1D dispatch
|
|
75
|
-
* @param totalThreads - Total number of threads needed
|
|
76
|
-
* @param workgroupSize - Threads per workgroup (default: 256)
|
|
77
|
-
* @returns Number of workgroups (rounded up)
|
|
78
|
-
*/
|
|
79
|
-
export declare function calculateWorkgroups1D(
|
|
80
|
-
totalThreads: number,
|
|
81
|
-
workgroupSize?: number
|
|
82
|
-
): number;
|
|
83
|
-
|
|
84
|
-
/**
|
|
85
|
-
* Calculate workgroup count for 2D dispatch
|
|
86
|
-
* @param width - Width dimension (e.g., matrix columns)
|
|
87
|
-
* @param height - Height dimension (e.g., matrix rows)
|
|
88
|
-
* @param tileSize - Tile size per workgroup (default: 16)
|
|
89
|
-
* @returns [workgroupsX, workgroupsY]
|
|
90
|
-
*/
|
|
91
|
-
export declare function calculateWorkgroups2D(
|
|
92
|
-
width: number,
|
|
93
|
-
height: number,
|
|
94
|
-
tileSize?: number
|
|
95
|
-
): [number, number];
|
|
96
|
-
|
|
97
|
-
/**
|
|
98
|
-
* Calculate workgroup count for 3D dispatch
|
|
99
|
-
* @param width - Width dimension
|
|
100
|
-
* @param height - Height dimension
|
|
101
|
-
* @param depth - Depth dimension
|
|
102
|
-
* @param tileSizeX - Tile size in X (default: 16)
|
|
103
|
-
* @param tileSizeY - Tile size in Y (default: 16)
|
|
104
|
-
* @param tileSizeZ - Tile size in Z (default: 1)
|
|
105
|
-
* @returns [workgroupsX, workgroupsY, workgroupsZ]
|
|
106
|
-
*/
|
|
107
|
-
export declare function calculateWorkgroups3D(
|
|
108
|
-
width: number,
|
|
109
|
-
height: number,
|
|
110
|
-
depth: number,
|
|
111
|
-
tileSizeX?: number,
|
|
112
|
-
tileSizeY?: number,
|
|
113
|
-
tileSizeZ?: number
|
|
114
|
-
): [number, number, number];
|
|
115
|
-
|
|
116
|
-
/**
|
|
117
|
-
* Dispatch options for advanced use cases
|
|
118
|
-
*/
|
|
119
|
-
export interface DispatchOptions {
|
|
120
|
-
/** Custom label for encoder and pass */
|
|
121
|
-
label?: string;
|
|
122
|
-
|
|
123
|
-
/** Bind groups (default: single group at index 0) */
|
|
124
|
-
bindGroups?: GPUBindGroup[];
|
|
125
|
-
|
|
126
|
-
/** Push constants (if supported) */
|
|
127
|
-
pushConstants?: ArrayBuffer;
|
|
128
|
-
|
|
129
|
-
/** Timestamp queries (if available) */
|
|
130
|
-
timestampWrites?: GPUComputePassTimestampWrites;
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
/**
|
|
134
|
-
* Advanced dispatch with full control
|
|
135
|
-
* Supports push constants, timestamps, and multiple bind groups
|
|
136
|
-
*/
|
|
137
|
-
export declare function dispatchAdvanced(
|
|
138
|
-
device: GPUDevice,
|
|
139
|
-
pipeline: GPUComputePipeline,
|
|
140
|
-
workgroups: number | [number, number, number],
|
|
141
|
-
options?: DispatchOptions
|
|
142
|
-
): void;
|
|
143
|
-
|
|
144
|
-
/**
|
|
145
|
-
* Batch multiple dispatches in a single command buffer
|
|
146
|
-
* Useful for multi-kernel operations that should be submitted together
|
|
147
|
-
*/
|
|
148
|
-
export declare function dispatchBatch(
|
|
149
|
-
device: GPUDevice,
|
|
150
|
-
batches: Array<{
|
|
151
|
-
pipeline: GPUComputePipeline;
|
|
152
|
-
bindGroup: GPUBindGroup;
|
|
153
|
-
workgroups: number | [number, number, number];
|
|
154
|
-
label?: string;
|
|
155
|
-
}>,
|
|
156
|
-
label?: string
|
|
157
|
-
): void;
|
|
@@ -116,120 +116,3 @@ export function recordDispatchIndirect(
|
|
|
116
116
|
}
|
|
117
117
|
|
|
118
118
|
|
|
119
|
-
export function dispatchMultiBindGroup(
|
|
120
|
-
device,
|
|
121
|
-
pipeline,
|
|
122
|
-
bindGroups,
|
|
123
|
-
workgroups,
|
|
124
|
-
label = 'compute'
|
|
125
|
-
) {
|
|
126
|
-
const [x, y, z] = assertWorkgroupLimits(device, workgroups, label);
|
|
127
|
-
const encoder = device.createCommandEncoder({ label: `${label}_encoder` });
|
|
128
|
-
const pass = encoder.beginComputePass({ label: `${label}_pass` });
|
|
129
|
-
pass.setPipeline(pipeline);
|
|
130
|
-
|
|
131
|
-
for (let i = 0; i < bindGroups.length; i++) {
|
|
132
|
-
pass.setBindGroup(i, bindGroups[i]);
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
pass.dispatchWorkgroups(x, y, z);
|
|
136
|
-
|
|
137
|
-
pass.end();
|
|
138
|
-
device.queue.submit([encoder.finish()]);
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
export function calculateWorkgroups1D(
|
|
143
|
-
totalThreads,
|
|
144
|
-
workgroupSize = 256
|
|
145
|
-
) {
|
|
146
|
-
return Math.ceil(totalThreads / workgroupSize);
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
export function calculateWorkgroups2D(
|
|
151
|
-
width,
|
|
152
|
-
height,
|
|
153
|
-
tileSize = 16
|
|
154
|
-
) {
|
|
155
|
-
return [
|
|
156
|
-
Math.ceil(width / tileSize),
|
|
157
|
-
Math.ceil(height / tileSize),
|
|
158
|
-
];
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
export function calculateWorkgroups3D(
|
|
163
|
-
width,
|
|
164
|
-
height,
|
|
165
|
-
depth,
|
|
166
|
-
tileSizeX = 16,
|
|
167
|
-
tileSizeY = 16,
|
|
168
|
-
tileSizeZ = 1
|
|
169
|
-
) {
|
|
170
|
-
return [
|
|
171
|
-
Math.ceil(width / tileSizeX),
|
|
172
|
-
Math.ceil(height / tileSizeY),
|
|
173
|
-
Math.ceil(depth / tileSizeZ),
|
|
174
|
-
];
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
export function dispatchAdvanced(
|
|
179
|
-
device,
|
|
180
|
-
pipeline,
|
|
181
|
-
workgroups,
|
|
182
|
-
options = {}
|
|
183
|
-
) {
|
|
184
|
-
const {
|
|
185
|
-
label = 'compute',
|
|
186
|
-
bindGroups = [],
|
|
187
|
-
timestampWrites,
|
|
188
|
-
} = options;
|
|
189
|
-
|
|
190
|
-
const [x, y, z] = assertWorkgroupLimits(device, workgroups, label);
|
|
191
|
-
const encoder = device.createCommandEncoder({ label: `${label}_encoder` });
|
|
192
|
-
|
|
193
|
-
const passDescriptor = {
|
|
194
|
-
label: `${label}_pass`,
|
|
195
|
-
};
|
|
196
|
-
|
|
197
|
-
if (timestampWrites) {
|
|
198
|
-
passDescriptor.timestampWrites = timestampWrites;
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
const pass = encoder.beginComputePass(passDescriptor);
|
|
202
|
-
pass.setPipeline(pipeline);
|
|
203
|
-
|
|
204
|
-
// Set bind groups
|
|
205
|
-
for (let i = 0; i < bindGroups.length; i++) {
|
|
206
|
-
pass.setBindGroup(i, bindGroups[i]);
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
// Dispatch
|
|
210
|
-
pass.dispatchWorkgroups(x, y, z);
|
|
211
|
-
|
|
212
|
-
pass.end();
|
|
213
|
-
device.queue.submit([encoder.finish()]);
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
export function dispatchBatch(
|
|
218
|
-
device,
|
|
219
|
-
batches,
|
|
220
|
-
label = 'batch'
|
|
221
|
-
) {
|
|
222
|
-
const encoder = device.createCommandEncoder({ label: `${label}_encoder` });
|
|
223
|
-
|
|
224
|
-
for (const batch of batches) {
|
|
225
|
-
const [x, y, z] = assertWorkgroupLimits(device, batch.workgroups, batch.label || label);
|
|
226
|
-
const pass = encoder.beginComputePass({ label: batch.label || `${label}_pass` });
|
|
227
|
-
pass.setPipeline(batch.pipeline);
|
|
228
|
-
pass.setBindGroup(0, batch.bindGroup);
|
|
229
|
-
pass.dispatchWorkgroups(x, y, z);
|
|
230
|
-
|
|
231
|
-
pass.end();
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
device.queue.submit([encoder.finish()]);
|
|
235
|
-
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/energy_eval.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// energy_eval_f16.wgsl
|
|
4
4
|
// Computes per-element energy contributions for f16 inputs.
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/energy_quintel_grad.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// energy_quintel_grad_f16.wgsl
|
|
4
4
|
// Quintel gradient kernel for f16 state: compute dE/d(state) into f32 output.
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/energy_quintel_reduce.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// energy_quintel_reduce_f16.wgsl
|
|
4
4
|
// Reduces quintel energy components and state sum per workgroup for f16 state.
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/energy_quintel_update.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// energy_quintel_update_f16.wgsl
|
|
4
4
|
// Quintel update kernel for f16 state.
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/energy_update.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// energy_update_f16.wgsl
|
|
4
4
|
// Gradient step on f16 state towards f16 target.
|
|
5
5
|
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
// JS dispatcher for fused_gate_up_gelu_gated_f16.wgsl.
|
|
2
|
+
// Computes output[M, N] = gelu_tanh(input @ W_gate) * (input @ W_up) in one pass.
|
|
3
|
+
// N = intermediate_size (per-layer; handles Gemma 4's double-wide MLP via uniform).
|
|
4
|
+
|
|
5
|
+
import { getDevice } from '../device.js';
|
|
6
|
+
import { acquireBuffer } from '../../memory/buffer-pool.js';
|
|
7
|
+
import { createTensor } from '../tensor.js';
|
|
8
|
+
import { KernelBase } from './kernel-base.js';
|
|
9
|
+
import { createUniformBufferWithView } from './utils.js';
|
|
10
|
+
import { releaseUniformBuffer } from '../uniform-cache.js';
|
|
11
|
+
import { getBuffer } from '../weight-buffer.js';
|
|
12
|
+
|
|
13
|
+
// Tile geometry tuned 2026-04-17 for RDNA3 (Radeon 8060S) + Gemma 4 E2B-scale
|
|
14
|
+
// FFN shapes (hidden=1536, intermediate=6144). COLS_PER_WG=64 keeps the total
|
|
15
|
+
// workgroup launch count in the low thousands for prefill M=64 — the smaller
|
|
16
|
+
// COLS_PER_WG=8 default used 49k workgroups and was occupancy-starved by
|
|
17
|
+
// launch/retire overhead.
|
|
18
|
+
const WORKGROUP_SIZE = 256;
|
|
19
|
+
const COLS_PER_WG = 64;
|
|
20
|
+
|
|
21
|
+
class FusedGateUpGeluKernel extends KernelBase {
|
|
22
|
+
async getPipeline(variant, constants = null) {
|
|
23
|
+
return this.getPipelineFor('matmul', variant, null, constants);
|
|
24
|
+
}
|
|
25
|
+
dispatch(pipeline, bindGroup, workgroups) {
|
|
26
|
+
this.dispatchKernel(pipeline, bindGroup, workgroups, 'fused_gate_up_gelu');
|
|
27
|
+
}
|
|
28
|
+
record(recorder, pipeline, bindGroup, workgroups) {
|
|
29
|
+
this.recordKernel(recorder, pipeline, bindGroup, workgroups, 'fused_gate_up_gelu');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
let cachedKernel = null;
|
|
34
|
+
function getKernel(device) {
|
|
35
|
+
if (!cachedKernel) cachedKernel = new FusedGateUpGeluKernel(device);
|
|
36
|
+
return cachedKernel;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function createFusedGateUpGeluUniform(device, recorder, params) {
|
|
40
|
+
return createUniformBufferWithView(
|
|
41
|
+
'fused_gate_up_gelu_uniforms',
|
|
42
|
+
32,
|
|
43
|
+
(view) => {
|
|
44
|
+
view.setUint32(0, params.M, true);
|
|
45
|
+
view.setUint32(4, params.hiddenSize, true);
|
|
46
|
+
view.setUint32(8, params.intermediateSize, true);
|
|
47
|
+
view.setUint32(12, params.transposeB ? 1 : 0, true);
|
|
48
|
+
view.setUint32(16, 0, true);
|
|
49
|
+
view.setUint32(20, 0, true);
|
|
50
|
+
view.setUint32(24, 0, true);
|
|
51
|
+
view.setUint32(28, 0, true);
|
|
52
|
+
},
|
|
53
|
+
recorder,
|
|
54
|
+
device
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async function executeFusedGateUpGelu(recorder, input, wGate, wUp, options) {
|
|
59
|
+
const {
|
|
60
|
+
M,
|
|
61
|
+
hiddenSize,
|
|
62
|
+
intermediateSize,
|
|
63
|
+
transposeB = true,
|
|
64
|
+
outputBuffer = null,
|
|
65
|
+
} = options;
|
|
66
|
+
|
|
67
|
+
if (!Number.isFinite(M) || !Number.isFinite(hiddenSize) || !Number.isFinite(intermediateSize)) {
|
|
68
|
+
throw new Error('[FusedGateUpGelu] M, hiddenSize, intermediateSize must be finite.');
|
|
69
|
+
}
|
|
70
|
+
if (input.dtype !== 'f16') {
|
|
71
|
+
throw new Error('[FusedGateUpGelu] input must be f16.');
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const device = getDevice();
|
|
75
|
+
if (!device) throw new Error('[FusedGateUpGelu] No GPU device.');
|
|
76
|
+
|
|
77
|
+
const kernel = getKernel(device);
|
|
78
|
+
const constants = { WORKGROUP_SIZE, COLS_PER_WG };
|
|
79
|
+
const pipeline = await kernel.getPipeline('gate_up_gelu_gated_tiled_f16', constants);
|
|
80
|
+
|
|
81
|
+
const outputSize = M * intermediateSize * 2; // f16
|
|
82
|
+
const output = outputBuffer || acquireBuffer(outputSize, undefined, 'fused_gate_up_gelu_output');
|
|
83
|
+
|
|
84
|
+
const uniform = createFusedGateUpGeluUniform(device, recorder, {
|
|
85
|
+
M, hiddenSize, intermediateSize, transposeB,
|
|
86
|
+
});
|
|
87
|
+
const wGateBuf = getBuffer(wGate);
|
|
88
|
+
const wUpBuf = getBuffer(wUp);
|
|
89
|
+
|
|
90
|
+
const bindGroup = device.createBindGroup({
|
|
91
|
+
label: 'fused_gate_up_gelu_bg',
|
|
92
|
+
layout: pipeline.getBindGroupLayout(0),
|
|
93
|
+
entries: [
|
|
94
|
+
{ binding: 0, resource: { buffer: uniform } },
|
|
95
|
+
{ binding: 1, resource: { buffer: input.buffer } },
|
|
96
|
+
{ binding: 2, resource: { buffer: wGateBuf } },
|
|
97
|
+
{ binding: 3, resource: { buffer: wUpBuf } },
|
|
98
|
+
{ binding: 4, resource: { buffer: output } },
|
|
99
|
+
],
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
const colTiles = Math.ceil(intermediateSize / COLS_PER_WG);
|
|
103
|
+
const workgroups = [colTiles, M, 1];
|
|
104
|
+
|
|
105
|
+
if (recorder) {
|
|
106
|
+
kernel.record(recorder, pipeline, bindGroup, workgroups);
|
|
107
|
+
} else {
|
|
108
|
+
kernel.dispatch(pipeline, bindGroup, workgroups);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
if (uniform) releaseUniformBuffer(uniform);
|
|
112
|
+
return createTensor(output, 'f16', [M, intermediateSize], 'fused_gate_up_gelu_output');
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export async function runFusedGateUpGelu(input, wGate, wUp, options = {}) {
|
|
116
|
+
return executeFusedGateUpGelu(null, input, wGate, wUp, options);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export async function recordFusedGateUpGelu(recorder, input, wGate, wUp, options = {}) {
|
|
120
|
+
return executeFusedGateUpGelu(recorder, input, wGate, wUp, options);
|
|
121
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/fused_ffn.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// Fused FFN Kernel - Native F16 I/O
|
|
4
4
|
//
|
|
5
5
|
// Full f16 pipeline variant: f16 input, f16 weights, f16 output.
|
|
@@ -110,25 +110,6 @@ fn unpack_q4_word(word: u32, nibble_shift: u32) -> vec4<u32> {
|
|
|
110
110
|
);
|
|
111
111
|
}
|
|
112
112
|
|
|
113
|
-
// Extract 4-bit quantized value from qs array
|
|
114
|
-
fn get_q4(qs: array<u32, 32>, idx: u32) -> u32 {
|
|
115
|
-
let chunk = idx / 64u;
|
|
116
|
-
let pos_in_chunk = idx % 64u;
|
|
117
|
-
let use_upper = pos_in_chunk >= 32u;
|
|
118
|
-
let byte_in_range = select(pos_in_chunk, pos_in_chunk - 32u, use_upper);
|
|
119
|
-
let byte_idx = chunk * 32u + byte_in_range;
|
|
120
|
-
|
|
121
|
-
let word_idx = byte_idx / 4u;
|
|
122
|
-
let byte_in_word = byte_idx % 4u;
|
|
123
|
-
let byte_val = (qs[word_idx] >> (byte_in_word * 8u)) & 0xFFu;
|
|
124
|
-
|
|
125
|
-
if (use_upper) {
|
|
126
|
-
return (byte_val >> 4u) & 0xFu;
|
|
127
|
-
} else {
|
|
128
|
-
return byte_val & 0xFu;
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
|
|
132
113
|
// SiLU activation: x * sigmoid(x)
|
|
133
114
|
fn silu(x: f32) -> f32 {
|
|
134
115
|
return x / (1.0 + exp(-x));
|
|
@@ -89,24 +89,6 @@ fn unpack_q4_word(word: u32, nibble_shift: u32) -> vec4<u32> {
|
|
|
89
89
|
);
|
|
90
90
|
}
|
|
91
91
|
|
|
92
|
-
fn get_q4(qs: array<u32, 32>, idx: u32) -> u32 {
|
|
93
|
-
let chunk = idx / 64u;
|
|
94
|
-
let pos_in_chunk = idx % 64u;
|
|
95
|
-
let use_upper = pos_in_chunk >= 32u;
|
|
96
|
-
let byte_in_range = select(pos_in_chunk, pos_in_chunk - 32u, use_upper);
|
|
97
|
-
let byte_idx = chunk * 32u + byte_in_range;
|
|
98
|
-
|
|
99
|
-
let word_idx = byte_idx / 4u;
|
|
100
|
-
let byte_in_word = byte_idx % 4u;
|
|
101
|
-
let byte_val = (qs[word_idx] >> (byte_in_word * 8u)) & 0xFFu;
|
|
102
|
-
|
|
103
|
-
if (use_upper) {
|
|
104
|
-
return (byte_val >> 4u) & 0xFu;
|
|
105
|
-
} else {
|
|
106
|
-
return byte_val & 0xFu;
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
|
|
110
92
|
fn silu(x: f32) -> f32 {
|
|
111
93
|
return x / (1.0 + exp(-x));
|
|
112
94
|
}
|