doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/gelu.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// GeLU Activation Kernel with F16 Input/Output
|
|
4
4
|
//
|
|
5
5
|
// F16 variant for reduced memory bandwidth when using F16 activations.
|
|
6
|
-
// Intermediate computations use F32 for precision.
|
|
7
6
|
|
|
8
7
|
enable f16;
|
|
9
8
|
|
|
@@ -23,12 +22,12 @@ struct Uniforms {
|
|
|
23
22
|
@group(0) @binding(2) var<storage, read_write> output: array<f16>;
|
|
24
23
|
@group(0) @binding(3) var<storage, read> gate: array<f16>;
|
|
25
24
|
|
|
26
|
-
fn gelu(x:
|
|
27
|
-
let sqrt_2_over_pi
|
|
28
|
-
let c
|
|
25
|
+
fn gelu(x: f16) -> f16 {
|
|
26
|
+
let sqrt_2_over_pi = f16(0.7978845608);
|
|
27
|
+
let c = f16(0.044715);
|
|
29
28
|
let inner = sqrt_2_over_pi * (x + c * x * x * x);
|
|
30
|
-
let inner_clamped = clamp(inner, -15.0, 15.0);
|
|
31
|
-
return 0.5 * x * (1.0 + tanh(inner_clamped));
|
|
29
|
+
let inner_clamped = clamp(inner, f16(-15.0), f16(15.0));
|
|
30
|
+
return f16(0.5) * x * (f16(1.0) + tanh(inner_clamped));
|
|
32
31
|
}
|
|
33
32
|
|
|
34
33
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
@@ -48,19 +47,19 @@ fn main(
|
|
|
48
47
|
let token_idx = idx / dim;
|
|
49
48
|
let dim_idx = idx % dim;
|
|
50
49
|
let row_base = token_idx * dim * 2u;
|
|
51
|
-
let g =
|
|
52
|
-
let up =
|
|
53
|
-
output[idx] =
|
|
50
|
+
let g = input[row_base + dim_idx];
|
|
51
|
+
let up = input[row_base + dim + dim_idx];
|
|
52
|
+
output[idx] = gelu(g) * up;
|
|
54
53
|
return;
|
|
55
54
|
}
|
|
56
55
|
|
|
57
56
|
if (HAS_GATE) {
|
|
58
|
-
let up =
|
|
59
|
-
let g =
|
|
60
|
-
output[idx] =
|
|
57
|
+
let up = input[idx];
|
|
58
|
+
let g = gate[idx];
|
|
59
|
+
output[idx] = gelu(g) * up;
|
|
61
60
|
return;
|
|
62
61
|
}
|
|
63
62
|
|
|
64
|
-
let x =
|
|
65
|
-
output[idx] =
|
|
63
|
+
let x = input[idx];
|
|
64
|
+
output[idx] = gelu(x);
|
|
66
65
|
}
|
|
@@ -11,10 +11,6 @@ export interface GroupNormOptions extends OutputBufferOptions {
|
|
|
11
11
|
eps: number;
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
-
export interface GroupNormSelectionContext {
|
|
15
|
-
isF16: boolean;
|
|
16
|
-
}
|
|
17
|
-
|
|
18
14
|
export declare function runGroupNorm(
|
|
19
15
|
input: Tensor,
|
|
20
16
|
weight: GPUBuffer | WeightBuffer,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/groupnorm_apply.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// GroupNorm Apply Kernel (NCHW, f16)
|
|
4
4
|
|
|
5
5
|
enable f16;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/groupnorm_stats.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// GroupNorm Stats Kernel (NCHW, f16 input)
|
|
4
4
|
|
|
5
5
|
enable f16;
|
|
@@ -49,7 +49,6 @@ export {
|
|
|
49
49
|
dequantize,
|
|
50
50
|
dequantizeRowwise,
|
|
51
51
|
dequantizeQ6K,
|
|
52
|
-
dequantizeQ8_0,
|
|
53
52
|
dequantizeMXFP4,
|
|
54
53
|
dequantizeMXFP4Expert,
|
|
55
54
|
recordDequantize,
|
|
@@ -149,6 +148,13 @@ export {
|
|
|
149
148
|
type ClampOptions,
|
|
150
149
|
} from './clamp.js';
|
|
151
150
|
|
|
151
|
+
// Static activation quantize/dequantize
|
|
152
|
+
export {
|
|
153
|
+
runActivationStaticQdq,
|
|
154
|
+
recordActivationStaticQdq,
|
|
155
|
+
type ActivationStaticQdqOptions,
|
|
156
|
+
} from './activation-static-qdq.js';
|
|
157
|
+
|
|
152
158
|
// Energy (EBM helpers)
|
|
153
159
|
export {
|
|
154
160
|
runEnergyEval,
|
|
@@ -270,10 +276,10 @@ export {
|
|
|
270
276
|
} from './sample.js';
|
|
271
277
|
|
|
272
278
|
export {
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
type
|
|
276
|
-
} from './
|
|
279
|
+
runLinearAttention,
|
|
280
|
+
recordLinearAttention,
|
|
281
|
+
type LinearAttentionOptions,
|
|
282
|
+
} from './linear_attention.js';
|
|
277
283
|
|
|
278
284
|
export {
|
|
279
285
|
runRepeatChannels,
|
|
@@ -362,6 +368,8 @@ export {
|
|
|
362
368
|
recordSoftmaxBackward,
|
|
363
369
|
runRmsNormBackward,
|
|
364
370
|
recordRmsNormBackward,
|
|
371
|
+
runLayerNormBackward,
|
|
372
|
+
recordLayerNormBackward,
|
|
365
373
|
runAttentionBackward,
|
|
366
374
|
recordAttentionBackward,
|
|
367
375
|
runRoPEBackward,
|
package/src/gpu/kernels/index.js
CHANGED
|
@@ -35,7 +35,6 @@ export {
|
|
|
35
35
|
dequantize,
|
|
36
36
|
dequantizeRowwise,
|
|
37
37
|
dequantizeQ6K,
|
|
38
|
-
dequantizeQ8_0,
|
|
39
38
|
dequantizeMXFP4,
|
|
40
39
|
dequantizeMXFP4Expert,
|
|
41
40
|
recordDequantize,
|
|
@@ -120,6 +119,12 @@ export {
|
|
|
120
119
|
recordClamp,
|
|
121
120
|
} from './clamp.js';
|
|
122
121
|
|
|
122
|
+
// Static activation quantize/dequantize
|
|
123
|
+
export {
|
|
124
|
+
runActivationStaticQdq,
|
|
125
|
+
recordActivationStaticQdq,
|
|
126
|
+
} from './activation-static-qdq.js';
|
|
127
|
+
|
|
123
128
|
// Energy (EBM helpers)
|
|
124
129
|
export {
|
|
125
130
|
runEnergyEval,
|
|
@@ -222,9 +227,9 @@ export {
|
|
|
222
227
|
} from './sample.js';
|
|
223
228
|
|
|
224
229
|
export {
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
} from './
|
|
230
|
+
runLinearAttention,
|
|
231
|
+
recordLinearAttention,
|
|
232
|
+
} from './linear_attention.js';
|
|
228
233
|
|
|
229
234
|
export {
|
|
230
235
|
runRepeatChannels,
|
|
@@ -5,13 +5,11 @@
|
|
|
5
5
|
|
|
6
6
|
enable f16;
|
|
7
7
|
|
|
8
|
-
const MAX_WORKGROUP_SIZE: u32 = 256u;
|
|
9
8
|
const MAX_HEAD_DIM: u32 = 256u;
|
|
10
9
|
|
|
11
10
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
12
11
|
override BIT_WIDTH: u32 = 4u;
|
|
13
12
|
override PACK_FACTOR: u32 = 8u;
|
|
14
|
-
override NUM_CENTROIDS: u32 = 16u;
|
|
15
13
|
override NUM_BOUNDARIES: u32 = 15u;
|
|
16
14
|
|
|
17
15
|
struct Uniforms {
|
|
@@ -7,13 +7,11 @@
|
|
|
7
7
|
|
|
8
8
|
enable f16;
|
|
9
9
|
|
|
10
|
-
const MAX_WORKGROUP_SIZE: u32 = 256u;
|
|
11
10
|
const MAX_HEAD_DIM: u32 = 256u;
|
|
12
11
|
|
|
13
12
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
14
13
|
override BIT_WIDTH_MSE: u32 = 3u;
|
|
15
14
|
override PACK_FACTOR_MSE: u32 = 10u;
|
|
16
|
-
override NUM_CENTROIDS_MSE: u32 = 8u;
|
|
17
15
|
override NUM_BOUNDARIES_MSE: u32 = 7u;
|
|
18
16
|
|
|
19
17
|
struct Uniforms {
|
|
@@ -13,10 +13,6 @@ export interface LayerNormOptions extends OutputBufferOptions {
|
|
|
13
13
|
normWeightDtype?: string;
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
-
export interface LayerNormSelectionContext {
|
|
17
|
-
isF16: boolean;
|
|
18
|
-
}
|
|
19
|
-
|
|
20
16
|
export declare function selectLayerNormKernel(options?: LayerNormOptions, isF16?: boolean): string;
|
|
21
17
|
|
|
22
18
|
export declare function runLayerNorm(
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/layernorm.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// LayerNorm Kernel (F16 Input/Output)
|
|
4
4
|
//
|
|
5
5
|
// LayerNorm(x) = (x - mean) / sqrt(var + eps) * weight + bias
|
|
@@ -2,7 +2,7 @@ import type { Tensor } from '../tensor.js';
|
|
|
2
2
|
import type { CommandRecorder } from '../command-recorder.js';
|
|
3
3
|
import type { OutputBufferOptions } from './types.js';
|
|
4
4
|
|
|
5
|
-
export interface
|
|
5
|
+
export interface LinearAttentionOptions extends OutputBufferOptions {
|
|
6
6
|
numHeads: number;
|
|
7
7
|
headDim: number;
|
|
8
8
|
numTokens?: number;
|
|
@@ -11,17 +11,17 @@ export interface SanaLinearAttentionOptions extends OutputBufferOptions {
|
|
|
11
11
|
summaryBuffer?: GPUBuffer | null;
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
-
export declare function
|
|
14
|
+
export declare function runLinearAttention(
|
|
15
15
|
query: Tensor,
|
|
16
16
|
key: Tensor,
|
|
17
17
|
value: Tensor,
|
|
18
|
-
options:
|
|
18
|
+
options: LinearAttentionOptions
|
|
19
19
|
): Promise<Tensor>;
|
|
20
20
|
|
|
21
|
-
export declare function
|
|
21
|
+
export declare function recordLinearAttention(
|
|
22
22
|
recorder: CommandRecorder,
|
|
23
23
|
query: Tensor,
|
|
24
24
|
key: Tensor,
|
|
25
25
|
value: Tensor,
|
|
26
|
-
options:
|
|
26
|
+
options: LinearAttentionOptions
|
|
27
27
|
): Promise<Tensor>;
|
|
@@ -5,14 +5,14 @@ import { unifiedKernelWrapper } from './utils.js';
|
|
|
5
5
|
import { selectRuleValue } from './rule-registry.js';
|
|
6
6
|
import { WORKGROUP_SIZES } from './constants.js';
|
|
7
7
|
|
|
8
|
-
function
|
|
9
|
-
return selectRuleValue('
|
|
8
|
+
function selectLinearAttentionVariant(isF16) {
|
|
9
|
+
return selectRuleValue('linearAttention', 'variant', { isF16 });
|
|
10
10
|
}
|
|
11
11
|
|
|
12
12
|
async function runSummary(target, query, key, value, summaryBuffer, uniforms, variant) {
|
|
13
13
|
const summarySize = uniforms.num_heads * (uniforms.head_dim + 1) * uniforms.head_dim;
|
|
14
14
|
await unifiedKernelWrapper(
|
|
15
|
-
'
|
|
15
|
+
'linear_attention_summary',
|
|
16
16
|
target,
|
|
17
17
|
variant,
|
|
18
18
|
[query, key, value, summaryBuffer],
|
|
@@ -30,7 +30,7 @@ async function runSummary(target, query, key, value, summaryBuffer, uniforms, va
|
|
|
30
30
|
|
|
31
31
|
async function runApply(target, query, summaryBuffer, outputBuffer, uniforms, variant) {
|
|
32
32
|
await unifiedKernelWrapper(
|
|
33
|
-
'
|
|
33
|
+
'linear_attention_apply',
|
|
34
34
|
target,
|
|
35
35
|
variant,
|
|
36
36
|
[query, summaryBuffer, outputBuffer],
|
|
@@ -48,11 +48,11 @@ async function runApply(target, query, summaryBuffer, outputBuffer, uniforms, va
|
|
|
48
48
|
);
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
async function
|
|
51
|
+
async function _linearAttention(target, query, key, value, options = {}) {
|
|
52
52
|
const recorder = target && typeof target.beginComputePass === 'function' ? target : null;
|
|
53
53
|
const device = target?.device || getDevice();
|
|
54
54
|
if (!device) {
|
|
55
|
-
throw new Error('
|
|
55
|
+
throw new Error('LinearAttention requires a WebGPU device.');
|
|
56
56
|
}
|
|
57
57
|
|
|
58
58
|
const {
|
|
@@ -73,23 +73,23 @@ async function _sanaLinearAttention(target, query, key, value, options = {}) {
|
|
|
73
73
|
!Number.isFinite(numTokens) ||
|
|
74
74
|
!Number.isFinite(hiddenSize)
|
|
75
75
|
) {
|
|
76
|
-
throw new Error('
|
|
76
|
+
throw new Error('LinearAttention requires numHeads, headDim, numTokens, and hiddenSize.');
|
|
77
77
|
}
|
|
78
78
|
if (hiddenSize !== numHeads * headDim) {
|
|
79
|
-
throw new Error(`
|
|
79
|
+
throw new Error(`LinearAttention hiddenSize mismatch: ${hiddenSize} != ${numHeads} * ${headDim}`);
|
|
80
80
|
}
|
|
81
81
|
|
|
82
82
|
const isF16 = query.dtype === 'f16';
|
|
83
|
-
const variant =
|
|
83
|
+
const variant = selectLinearAttentionVariant(isF16);
|
|
84
84
|
const temporarySummary = summaryBuffer || acquireBuffer(
|
|
85
85
|
numHeads * (headDim + 1) * headDim * Float32Array.BYTES_PER_ELEMENT,
|
|
86
86
|
undefined,
|
|
87
|
-
'
|
|
87
|
+
'linear_attention_summary'
|
|
88
88
|
);
|
|
89
89
|
const output = outputBuffer || acquireBuffer(
|
|
90
90
|
numTokens * hiddenSize * dtypeBytes(query.dtype),
|
|
91
91
|
undefined,
|
|
92
|
-
'
|
|
92
|
+
'linear_attention_output'
|
|
93
93
|
);
|
|
94
94
|
|
|
95
95
|
const uniforms = {
|
|
@@ -103,7 +103,7 @@ async function _sanaLinearAttention(target, query, key, value, options = {}) {
|
|
|
103
103
|
try {
|
|
104
104
|
await runSummary(target, query, key, value, temporarySummary, uniforms, variant);
|
|
105
105
|
await runApply(target, query, temporarySummary, output, uniforms, variant);
|
|
106
|
-
return createTensor(output, query.dtype, [numTokens, hiddenSize], '
|
|
106
|
+
return createTensor(output, query.dtype, [numTokens, hiddenSize], 'linear_attention_output');
|
|
107
107
|
} catch (error) {
|
|
108
108
|
if (ownsOutput) {
|
|
109
109
|
releaseBuffer(output);
|
|
@@ -120,10 +120,10 @@ async function _sanaLinearAttention(target, query, key, value, options = {}) {
|
|
|
120
120
|
}
|
|
121
121
|
}
|
|
122
122
|
|
|
123
|
-
export async function
|
|
124
|
-
return
|
|
123
|
+
export async function runLinearAttention(query, key, value, options = {}) {
|
|
124
|
+
return _linearAttention(null, query, key, value, options);
|
|
125
125
|
}
|
|
126
126
|
|
|
127
|
-
export async function
|
|
128
|
-
return
|
|
127
|
+
export async function recordLinearAttention(recorder, query, key, value, options = {}) {
|
|
128
|
+
return _linearAttention(recorder, query, key, value, options);
|
|
129
129
|
}
|
|
@@ -68,30 +68,6 @@ export declare class LogitMergeKernel {
|
|
|
68
68
|
): Promise<GPUBuffer>;
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
-
/**
|
|
72
|
-
* Get the logit merge kernel instance.
|
|
73
|
-
*/
|
|
74
|
-
export declare function getLogitMergeKernel(): LogitMergeKernel;
|
|
75
|
-
|
|
76
|
-
/**
|
|
77
|
-
* Merge two logit buffers using weighted averaging.
|
|
78
|
-
* Convenience function that uses the singleton kernel.
|
|
79
|
-
*
|
|
80
|
-
* @param logitsA - First logit buffer
|
|
81
|
-
* @param logitsB - Second logit buffer
|
|
82
|
-
* @param vocabSize - Vocabulary size
|
|
83
|
-
* @param weights - Merge weights
|
|
84
|
-
* @param temperature - Temperature scaling
|
|
85
|
-
* @returns Merged logit buffer
|
|
86
|
-
*/
|
|
87
|
-
export declare function mergeLogits(
|
|
88
|
-
logitsA: GPUBuffer,
|
|
89
|
-
logitsB: GPUBuffer,
|
|
90
|
-
vocabSize: number,
|
|
91
|
-
weights: number[],
|
|
92
|
-
temperature: number
|
|
93
|
-
): Promise<GPUBuffer>;
|
|
94
|
-
|
|
95
71
|
/**
|
|
96
72
|
* Merge multiple logit buffers.
|
|
97
73
|
* Convenience function that uses the singleton kernel.
|
|
@@ -354,7 +354,7 @@ function normalizeWeights(weights, expectedCount) {
|
|
|
354
354
|
let _instance = null;
|
|
355
355
|
|
|
356
356
|
|
|
357
|
-
|
|
357
|
+
function getLogitMergeKernel() {
|
|
358
358
|
if (!_instance) {
|
|
359
359
|
_instance = new LogitMergeKernel();
|
|
360
360
|
}
|
|
@@ -362,22 +362,6 @@ export function getLogitMergeKernel() {
|
|
|
362
362
|
}
|
|
363
363
|
|
|
364
364
|
|
|
365
|
-
export async function mergeLogits(logitsA, logitsB, vocabSize, weights, temperature) {
|
|
366
|
-
if (!weights) {
|
|
367
|
-
throw new Error('LogitMerge weights are required.');
|
|
368
|
-
}
|
|
369
|
-
if (temperature == null) {
|
|
370
|
-
throw new Error('LogitMerge temperature is required.');
|
|
371
|
-
}
|
|
372
|
-
const kernel = getLogitMergeKernel();
|
|
373
|
-
return kernel.merge(logitsA, logitsB, vocabSize, {
|
|
374
|
-
strategy: 'weighted',
|
|
375
|
-
weights,
|
|
376
|
-
temperature,
|
|
377
|
-
});
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
|
|
381
365
|
export async function mergeMultipleLogits(logitBuffers, vocabSize, weights, temperature) {
|
|
382
366
|
if (!weights) {
|
|
383
367
|
throw new Error('LogitMerge weights are required.');
|
|
@@ -96,7 +96,7 @@ export function calculateMatmulDispatch(variant, useQ4KFused, useGemv, M, N, con
|
|
|
96
96
|
}
|
|
97
97
|
|
|
98
98
|
|
|
99
|
-
export function createMatmulUniformBuffer(label, M, N, K, alpha, useQ4KFused, transposeB, uniformWorkgroupsX, recorder, device) {
|
|
99
|
+
export function createMatmulUniformBuffer(label, M, N, K, alpha, useQ4KFused, transposeB, uniformWorkgroupsX, recorder, device, extras = null) {
|
|
100
100
|
// Shader struct is 32 bytes: M, N, K, alpha, transpose_b/num_blocks, workgroups_x/_pad0, _pad1, _pad2
|
|
101
101
|
const uniformSize = 32;
|
|
102
102
|
|
|
@@ -117,6 +117,12 @@ export function createMatmulUniformBuffer(label, M, N, K, alpha, useQ4KFused, tr
|
|
|
117
117
|
// workgroups_x (or _pad0 if not needed)
|
|
118
118
|
view.setUint32(20, uniformWorkgroupsX ?? 0, true);
|
|
119
119
|
// _pad1, _pad2 - leave as zeros (already zero-initialized)
|
|
120
|
+
// Extras for fused-rmsnorm WideTile variant: eps f32 overwrites slot 20
|
|
121
|
+
// (workgroupsX is unused for those variants). rmsNormOffset is a kernel
|
|
122
|
+
// override constant, not a uniform field.
|
|
123
|
+
if (extras && Number.isFinite(extras.eps)) {
|
|
124
|
+
view.setFloat32(20, extras.eps, true);
|
|
125
|
+
}
|
|
120
126
|
},
|
|
121
127
|
recorder,
|
|
122
128
|
device
|
|
@@ -441,9 +441,76 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
|
|
|
441
441
|
const fusedAllowed = !fusedQ4KDisabled;
|
|
442
442
|
const isQ4K = bDtype === 'q4k';
|
|
443
443
|
const wantF16Output = requestedOutputDtype === 'f16' && capabilities.hasF16;
|
|
444
|
-
|
|
444
|
+
let q4kVariant = isQ4K && capabilities.hasSubgroups && fusedAllowed
|
|
445
445
|
? selectQ4KFusedVariant(M === 1, wantF16Output, aDtype, phase)
|
|
446
446
|
: null;
|
|
447
|
+
// Opt-in override: when useTiledQ4KPrefill is set on options, replace the
|
|
448
|
+
// default batched f16-output variant with the register-tiled variant
|
|
449
|
+
// whenever the shape (Q4_K weights, f16 output, prefill M>=16) and
|
|
450
|
+
// capability (hasF16) match. Matches both batched_f16 (f32 activations)
|
|
451
|
+
// and batched_f16a (f16 activations); the tiled kernel takes f32 A and
|
|
452
|
+
// f16 C per the q4_fused_batched_f16 binding contract. Decoupled from
|
|
453
|
+
// subgroups because the tiled kernel does not use subgroup reduction.
|
|
454
|
+
if (
|
|
455
|
+
q4kVariant === 'q4_fused_batched_f16'
|
|
456
|
+
&& options.useTiledQ4KPrefill === true
|
|
457
|
+
&& phase === 'prefill'
|
|
458
|
+
&& M >= 16
|
|
459
|
+
&& aDtype === 'f32'
|
|
460
|
+
&& wantF16Output
|
|
461
|
+
&& capabilities.hasF16 === true
|
|
462
|
+
) {
|
|
463
|
+
q4kVariant = 'q4_fused_prefill_tiled_f16';
|
|
464
|
+
}
|
|
465
|
+
// WideTile override (ported from ORT MatMulNBitsWideTile): register-tiled,
|
|
466
|
+
// 1 thread per output column × TILE_M rows accumulated in registers.
|
|
467
|
+
// Orthogonal to useTiledQ4KPrefill — if both flags are set, WideTile wins
|
|
468
|
+
// because it has materially fewer workgroups at Gemma-4-scale prefill.
|
|
469
|
+
// Routes to the f16-output or f32-output WideTile variant depending on
|
|
470
|
+
// what the caller asked for (Gemma 4's FFN picks q4_fused_batched with
|
|
471
|
+
// f32 output; attention projections may want f16).
|
|
472
|
+
if (
|
|
473
|
+
options.useWideTileQ4KPrefill === true
|
|
474
|
+
&& phase === 'prefill'
|
|
475
|
+
&& M >= 4
|
|
476
|
+
&& aDtype === 'f32'
|
|
477
|
+
&& capabilities.hasF16 === true
|
|
478
|
+
) {
|
|
479
|
+
if (
|
|
480
|
+
wantF16Output
|
|
481
|
+
&& (q4kVariant === 'q4_fused_batched_f16' || q4kVariant === 'q4_fused_prefill_tiled_f16')
|
|
482
|
+
) {
|
|
483
|
+
q4kVariant = 'q4_fused_widetile_f16';
|
|
484
|
+
} else if (
|
|
485
|
+
!wantF16Output
|
|
486
|
+
&& q4kVariant === 'q4_fused_batched'
|
|
487
|
+
) {
|
|
488
|
+
q4kVariant = 'q4_fused_widetile';
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
// Residual-fused WideTile override: when the caller passes a residualTensor
|
|
492
|
+
// and opts into the fusion via useWideTileResidualFusion, route the
|
|
493
|
+
// f32-output WideTile variant to its residual-epilogue twin. Saves one
|
|
494
|
+
// downstream doResidualAdd dispatch per fusion point.
|
|
495
|
+
if (
|
|
496
|
+
options.useWideTileResidualFusion === true
|
|
497
|
+
&& options.residualTensor != null
|
|
498
|
+
&& q4kVariant === 'q4_fused_widetile'
|
|
499
|
+
) {
|
|
500
|
+
q4kVariant = 'q4_fused_widetile_residual';
|
|
501
|
+
}
|
|
502
|
+
// RMSNorm-fused WideTile override: when the caller passes normWeight and
|
|
503
|
+
// opts into the fusion via useFusedRmsnormWideTile, route the f32-output
|
|
504
|
+
// WideTile variant to its rmsnorm-prologue twin. Each q/k/v or gate/up
|
|
505
|
+
// call runs the norm internally; upstream standalone rmsnorm is skipped
|
|
506
|
+
// by the caller when all downstream matmuls at that site use this variant.
|
|
507
|
+
if (
|
|
508
|
+
options.useFusedRmsnormWideTile === true
|
|
509
|
+
&& options.normWeight != null
|
|
510
|
+
&& q4kVariant === 'q4_fused_widetile'
|
|
511
|
+
) {
|
|
512
|
+
q4kVariant = 'q4_fused_rmsnorm_widetile';
|
|
513
|
+
}
|
|
447
514
|
|
|
448
515
|
const effectiveBDtype = bDtype === 'q4k' ? 'f32' : bDtype;
|
|
449
516
|
const matmulVariant = selectMatmulKernel({
|
|
@@ -48,34 +48,6 @@ export interface MatmulOptions extends OutputBufferOptions, OutputDtypeOptions,
|
|
|
48
48
|
executionPolicies?: ExecutionV1PoliciesSchema | null;
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
/** Context for base matmul kernel selection rules. */
|
|
52
|
-
export interface MatmulKernelSelectionContext {
|
|
53
|
-
useF16Matmul: boolean;
|
|
54
|
-
useF16wF32a: boolean;
|
|
55
|
-
useVec4: boolean;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
/** Context for fused Q4K variant selection rules. */
|
|
59
|
-
export interface MatmulQ4KFusedContext {
|
|
60
|
-
useF16A: boolean;
|
|
61
|
-
useF16Out: boolean;
|
|
62
|
-
isM1: boolean;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
/** Context for GEMV variant selection rules. */
|
|
66
|
-
export interface GemvSelectionContext {
|
|
67
|
-
hasSubgroups: boolean;
|
|
68
|
-
useF16Gemv: boolean;
|
|
69
|
-
useF32Gemv: boolean;
|
|
70
|
-
useMulticol: boolean;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
/** Context for top-level matmul variant selection rules. */
|
|
74
|
-
export interface MatmulVariantSelectionContext {
|
|
75
|
-
canFused: boolean;
|
|
76
|
-
useGemv: boolean;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
51
|
/**
|
|
80
52
|
* Check if fused Q4K kernels are disabled.
|
|
81
53
|
*/
|