doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/rope.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// Rotary Position Embeddings (RoPE) Kernel (F16)
|
|
4
4
|
//
|
|
5
5
|
// Applies rotary position embeddings to Q and K tensors.
|
|
6
6
|
// Same math as rope.wgsl, but input/output are f16.
|
|
7
|
-
//
|
|
7
|
+
// The main entry keeps apply-time arithmetic in f16 for the all-f16 lane.
|
|
8
8
|
//
|
|
9
9
|
// Supports:
|
|
10
10
|
// - Original RoPE (base = 10000)
|
|
@@ -15,6 +15,11 @@ enable f16;
|
|
|
15
15
|
|
|
16
16
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
17
17
|
|
|
18
|
+
// YaRN parameters (model-family-specific, config-driven)
|
|
19
|
+
override YARN_BETA_FAST: f32 = 32.0;
|
|
20
|
+
override YARN_BETA_SLOW: f32 = 1.0;
|
|
21
|
+
override YARN_ALPHA: f32 = 1.0;
|
|
22
|
+
|
|
18
23
|
// Mathematical constant
|
|
19
24
|
const PI: f32 = 3.14159265359;
|
|
20
25
|
|
|
@@ -27,6 +32,10 @@ struct Uniforms {
|
|
|
27
32
|
rope_scale: f32,
|
|
28
33
|
rotary_dim: u32,
|
|
29
34
|
interleaved: u32,
|
|
35
|
+
pair_span_dim: u32,
|
|
36
|
+
_pad0: u32,
|
|
37
|
+
_pad1: u32,
|
|
38
|
+
_pad2: u32,
|
|
30
39
|
}
|
|
31
40
|
|
|
32
41
|
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
@@ -41,11 +50,11 @@ fn get_first_rotary_idx(pair_idx: u32) -> u32 {
|
|
|
41
50
|
return pair_idx;
|
|
42
51
|
}
|
|
43
52
|
|
|
44
|
-
fn get_second_rotary_idx(pair_idx: u32,
|
|
53
|
+
fn get_second_rotary_idx(pair_idx: u32, pair_span_dim: u32) -> u32 {
|
|
45
54
|
if (u.interleaved == 1u) {
|
|
46
55
|
return pair_idx * 2u + 1u;
|
|
47
56
|
}
|
|
48
|
-
return pair_idx + (
|
|
57
|
+
return pair_idx + (pair_span_dim / 2u);
|
|
49
58
|
}
|
|
50
59
|
|
|
51
60
|
// Apply RoPE using precomputed frequencies
|
|
@@ -74,20 +83,20 @@ fn main(
|
|
|
74
83
|
let actual_pos = start_pos + pos;
|
|
75
84
|
|
|
76
85
|
let freq_idx = actual_pos * half_dim + pair_idx;
|
|
77
|
-
let cos_val = freqs_cos[freq_idx];
|
|
78
|
-
let sin_val = freqs_sin[freq_idx];
|
|
86
|
+
let cos_val = f16(freqs_cos[freq_idx]);
|
|
87
|
+
let sin_val = f16(freqs_sin[freq_idx]);
|
|
79
88
|
|
|
80
89
|
let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
|
|
81
90
|
let first_idx = get_first_rotary_idx(pair_idx);
|
|
82
|
-
let second_idx = get_second_rotary_idx(pair_idx,
|
|
83
|
-
let x0 =
|
|
84
|
-
let x1 =
|
|
91
|
+
let second_idx = get_second_rotary_idx(pair_idx, u.pair_span_dim);
|
|
92
|
+
let x0 = input[base_idx + first_idx];
|
|
93
|
+
let x1 = input[base_idx + second_idx];
|
|
85
94
|
|
|
86
95
|
let y0 = x0 * cos_val - x1 * sin_val;
|
|
87
96
|
let y1 = x0 * sin_val + x1 * cos_val;
|
|
88
97
|
|
|
89
|
-
input[base_idx + first_idx] =
|
|
90
|
-
input[base_idx + second_idx] =
|
|
98
|
+
input[base_idx + first_idx] = y0;
|
|
99
|
+
input[base_idx + second_idx] = y1;
|
|
91
100
|
}
|
|
92
101
|
|
|
93
102
|
// Compute frequencies on-the-fly (no precomputation needed)
|
|
@@ -102,6 +111,7 @@ fn rope_compute_freqs(
|
|
|
102
111
|
let rope_base = u.rope_base;
|
|
103
112
|
let rope_scale = u.rope_scale;
|
|
104
113
|
let rotary_dim = u.rotary_dim;
|
|
114
|
+
let pair_span_dim = u.pair_span_dim;
|
|
105
115
|
|
|
106
116
|
let idx = global_id.x;
|
|
107
117
|
let half_dim = rotary_dim / 2u;
|
|
@@ -118,7 +128,7 @@ fn rope_compute_freqs(
|
|
|
118
128
|
|
|
119
129
|
let actual_pos = f32(start_pos + pos) / rope_scale;
|
|
120
130
|
|
|
121
|
-
let exponent = f32(pair_idx * 2u) / f32(
|
|
131
|
+
let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
|
|
122
132
|
let freq = 1.0 / pow(rope_base, exponent);
|
|
123
133
|
let theta = actual_pos * freq;
|
|
124
134
|
|
|
@@ -127,7 +137,7 @@ fn rope_compute_freqs(
|
|
|
127
137
|
|
|
128
138
|
let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
|
|
129
139
|
let first_idx = get_first_rotary_idx(pair_idx);
|
|
130
|
-
let second_idx = get_second_rotary_idx(pair_idx,
|
|
140
|
+
let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
|
|
131
141
|
let x0 = f32(input[base_idx + first_idx]);
|
|
132
142
|
let x1 = f32(input[base_idx + second_idx]);
|
|
133
143
|
|
|
@@ -150,6 +160,7 @@ fn rope_qk(
|
|
|
150
160
|
let rope_base = u.rope_base;
|
|
151
161
|
let rope_scale = u.rope_scale;
|
|
152
162
|
let rotary_dim = u.rotary_dim;
|
|
163
|
+
let pair_span_dim = u.pair_span_dim;
|
|
153
164
|
|
|
154
165
|
let idx = global_id.x;
|
|
155
166
|
let half_dim = rotary_dim / 2u;
|
|
@@ -166,7 +177,7 @@ fn rope_qk(
|
|
|
166
177
|
|
|
167
178
|
let actual_pos = f32(start_pos + pos) / rope_scale;
|
|
168
179
|
|
|
169
|
-
let exponent = f32(pair_idx * 2u) / f32(
|
|
180
|
+
let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
|
|
170
181
|
let freq = 1.0 / pow(rope_base, exponent);
|
|
171
182
|
let theta = actual_pos * freq;
|
|
172
183
|
|
|
@@ -177,7 +188,7 @@ fn rope_qk(
|
|
|
177
188
|
let k_base_idx = q_base_idx + head_dim;
|
|
178
189
|
|
|
179
190
|
let first_idx = get_first_rotary_idx(pair_idx);
|
|
180
|
-
let second_idx = get_second_rotary_idx(pair_idx,
|
|
191
|
+
let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
|
|
181
192
|
let q0 = f32(input[q_base_idx + first_idx]);
|
|
182
193
|
let q1 = f32(input[q_base_idx + second_idx]);
|
|
183
194
|
input[q_base_idx + first_idx] = f16(q0 * cos_val - q1 * sin_val);
|
|
@@ -198,6 +209,7 @@ fn precompute_freqs(
|
|
|
198
209
|
let rope_base = u.rope_base;
|
|
199
210
|
let rope_scale = u.rope_scale;
|
|
200
211
|
let rotary_dim = u.rotary_dim;
|
|
212
|
+
let pair_span_dim = u.pair_span_dim;
|
|
201
213
|
|
|
202
214
|
let idx = global_id.x;
|
|
203
215
|
let half_dim = rotary_dim / 2u;
|
|
@@ -211,7 +223,7 @@ fn precompute_freqs(
|
|
|
211
223
|
let dim_idx = idx % half_dim;
|
|
212
224
|
|
|
213
225
|
let actual_pos = f32(pos) / rope_scale;
|
|
214
|
-
let exponent = f32(dim_idx * 2u) / f32(
|
|
226
|
+
let exponent = f32(dim_idx * 2u) / f32(pair_span_dim);
|
|
215
227
|
let freq = 1.0 / pow(rope_base, exponent);
|
|
216
228
|
let theta = actual_pos * freq;
|
|
217
229
|
|
|
@@ -230,6 +242,7 @@ fn rope_ntk_scaled(
|
|
|
230
242
|
let start_pos = u.start_pos;
|
|
231
243
|
var rope_base = u.rope_base;
|
|
232
244
|
let rope_scale = u.rope_scale;
|
|
245
|
+
let pair_span_dim = u.pair_span_dim;
|
|
233
246
|
|
|
234
247
|
let idx = global_id.x;
|
|
235
248
|
let half_dim = rotary_dim / 2u;
|
|
@@ -239,7 +252,7 @@ fn rope_ntk_scaled(
|
|
|
239
252
|
return;
|
|
240
253
|
}
|
|
241
254
|
|
|
242
|
-
rope_base = rope_base * pow(rope_scale, f32(
|
|
255
|
+
rope_base = rope_base * pow(rope_scale, f32(pair_span_dim) / (f32(pair_span_dim) - 2.0));
|
|
243
256
|
|
|
244
257
|
let pos = idx / (num_heads * half_dim);
|
|
245
258
|
let remainder = idx % (num_heads * half_dim);
|
|
@@ -248,7 +261,7 @@ fn rope_ntk_scaled(
|
|
|
248
261
|
|
|
249
262
|
let actual_pos = f32(start_pos + pos);
|
|
250
263
|
|
|
251
|
-
let exponent = f32(pair_idx * 2u) / f32(
|
|
264
|
+
let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
|
|
252
265
|
let freq = 1.0 / pow(rope_base, exponent);
|
|
253
266
|
let theta = actual_pos * freq;
|
|
254
267
|
|
|
@@ -257,7 +270,7 @@ fn rope_ntk_scaled(
|
|
|
257
270
|
|
|
258
271
|
let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
|
|
259
272
|
let first_idx = get_first_rotary_idx(pair_idx);
|
|
260
|
-
let second_idx = get_second_rotary_idx(pair_idx,
|
|
273
|
+
let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
|
|
261
274
|
let x0 = f32(input[base_idx + first_idx]);
|
|
262
275
|
let x1 = f32(input[base_idx + second_idx]);
|
|
263
276
|
|
|
@@ -276,6 +289,7 @@ fn rope_yarn(
|
|
|
276
289
|
let start_pos = u.start_pos;
|
|
277
290
|
let rope_base = u.rope_base;
|
|
278
291
|
let rope_scale = u.rope_scale;
|
|
292
|
+
let pair_span_dim = u.pair_span_dim;
|
|
279
293
|
|
|
280
294
|
let idx = global_id.x;
|
|
281
295
|
let half_dim = rotary_dim / 2u;
|
|
@@ -292,17 +306,17 @@ fn rope_yarn(
|
|
|
292
306
|
|
|
293
307
|
let actual_pos = f32(start_pos + pos);
|
|
294
308
|
|
|
295
|
-
let beta_fast: f32 =
|
|
296
|
-
let beta_slow: f32 =
|
|
309
|
+
let beta_fast: f32 = YARN_BETA_FAST;
|
|
310
|
+
let beta_slow: f32 = YARN_BETA_SLOW;
|
|
297
311
|
|
|
298
|
-
let exponent = f32(pair_idx * 2u) / f32(
|
|
312
|
+
let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
|
|
299
313
|
let orig_freq = 1.0 / pow(rope_base, exponent);
|
|
300
314
|
|
|
301
315
|
let wavelength = 2.0 * PI / orig_freq;
|
|
302
316
|
|
|
303
317
|
var ramp: f32;
|
|
304
|
-
let low_wavelength = f32(
|
|
305
|
-
let high_wavelength = f32(
|
|
318
|
+
let low_wavelength = f32(pair_span_dim) / beta_fast;
|
|
319
|
+
let high_wavelength = f32(pair_span_dim) / beta_slow;
|
|
306
320
|
|
|
307
321
|
if (wavelength < low_wavelength) {
|
|
308
322
|
ramp = 0.0;
|
|
@@ -321,7 +335,7 @@ fn rope_yarn(
|
|
|
321
335
|
|
|
322
336
|
let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
|
|
323
337
|
let first_idx = get_first_rotary_idx(pair_idx);
|
|
324
|
-
let second_idx = get_second_rotary_idx(pair_idx,
|
|
338
|
+
let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
|
|
325
339
|
let x0 = f32(input[base_idx + first_idx]);
|
|
326
340
|
let x1 = f32(input[base_idx + second_idx]);
|
|
327
341
|
|
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
20
20
|
const MAX_WORKGROUP_SIZE: u32 = 256u;
|
|
21
21
|
const MAX_TOP_K: u32 = 128u; // Max top-k supported
|
|
22
|
+
const NEG_INF: f32 = -3.402823e+38;
|
|
22
23
|
|
|
23
24
|
struct Uniforms {
|
|
24
25
|
vocab_size: u32,
|
|
@@ -78,7 +79,7 @@ fn find_topk_phase1(
|
|
|
78
79
|
|
|
79
80
|
// Single workgroup: write all logits directly for exact top-k
|
|
80
81
|
if (num_wg.x == 1u) {
|
|
81
|
-
var val: f32 =
|
|
82
|
+
var val: f32 = NEG_INF;
|
|
82
83
|
if (global_idx < vocab_size && global_idx != pad_id) {
|
|
83
84
|
val = apply_softcap(logits[global_idx], softcap) / temperature;
|
|
84
85
|
}
|
|
@@ -88,7 +89,7 @@ fn find_topk_phase1(
|
|
|
88
89
|
}
|
|
89
90
|
|
|
90
91
|
// Each thread finds max in its assigned range
|
|
91
|
-
var local_max: f32 =
|
|
92
|
+
var local_max: f32 = NEG_INF; // -FLT_MAX
|
|
92
93
|
var local_max_idx: u32 = 0u;
|
|
93
94
|
|
|
94
95
|
// Stride through vocabulary
|
|
@@ -152,7 +153,7 @@ fn find_topk_phase2(
|
|
|
152
153
|
shared_values[thread_idx] = topk_logits[thread_idx];
|
|
153
154
|
shared_indices[thread_idx] = topk_indices[thread_idx];
|
|
154
155
|
} else {
|
|
155
|
-
shared_values[thread_idx] =
|
|
156
|
+
shared_values[thread_idx] = NEG_INF;
|
|
156
157
|
shared_indices[thread_idx] = 0u;
|
|
157
158
|
}
|
|
158
159
|
}
|
|
@@ -256,7 +257,7 @@ fn sample_single_pass(
|
|
|
256
257
|
let softcap = u.logit_softcap;
|
|
257
258
|
|
|
258
259
|
// Phase 1: Find global max
|
|
259
|
-
var local_max: f32 =
|
|
260
|
+
var local_max: f32 = NEG_INF;
|
|
260
261
|
var local_max_idx: u32 = 0u;
|
|
261
262
|
|
|
262
263
|
var idx = gid.x;
|
|
@@ -320,7 +321,7 @@ fn argmax(
|
|
|
320
321
|
let softcap = u.logit_softcap;
|
|
321
322
|
|
|
322
323
|
// Each thread finds max in its chunk
|
|
323
|
-
var local_max: f32 =
|
|
324
|
+
var local_max: f32 = NEG_INF;
|
|
324
325
|
var local_max_idx: u32 = 0u;
|
|
325
326
|
|
|
326
327
|
var idx = global_idx;
|
|
@@ -378,7 +379,7 @@ fn argmax_reduce(
|
|
|
378
379
|
shared_values[thread_idx] = topk_logits[thread_idx];
|
|
379
380
|
shared_indices[thread_idx] = topk_indices[thread_idx];
|
|
380
381
|
} else {
|
|
381
|
-
shared_values[thread_idx] =
|
|
382
|
+
shared_values[thread_idx] = NEG_INF;
|
|
382
383
|
shared_indices[thread_idx] = 0u;
|
|
383
384
|
}
|
|
384
385
|
workgroupBarrier();
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/sample.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
3
|
-
// AUTO-GENERATED from src/gpu/kernels/sample.wgsl.
|
|
4
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
5
3
|
// sample_f16.wgsl
|
|
6
4
|
|
|
7
5
|
/**
|
|
8
6
|
* GPU-Side Sampling Kernel (f16 logits)
|
|
9
7
|
*
|
|
10
8
|
* Same as sample.wgsl but logits are f16.
|
|
11
|
-
*
|
|
9
|
+
* The all-f16 lane keeps sampling comparisons and reductions in f16.
|
|
12
10
|
*/
|
|
13
11
|
|
|
14
12
|
enable f16;
|
|
@@ -17,6 +15,7 @@ enable f16;
|
|
|
17
15
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
18
16
|
const MAX_WORKGROUP_SIZE: u32 = 256u;
|
|
19
17
|
const MAX_TOP_K: u32 = 128u;
|
|
18
|
+
const NEG_INF = f16(-65504.0);
|
|
20
19
|
|
|
21
20
|
struct Uniforms {
|
|
22
21
|
vocab_size: u32,
|
|
@@ -29,14 +28,14 @@ struct Uniforms {
|
|
|
29
28
|
pad0: u32,
|
|
30
29
|
}
|
|
31
30
|
|
|
32
|
-
fn apply_softcap(x:
|
|
33
|
-
if (softcap <= 0.0) {
|
|
31
|
+
fn apply_softcap(x: f16, softcap: f16) -> f16 {
|
|
32
|
+
if (softcap <= f16(0.0)) {
|
|
34
33
|
return x;
|
|
35
34
|
}
|
|
36
35
|
return softcap * tanh(x / softcap);
|
|
37
36
|
}
|
|
38
37
|
|
|
39
|
-
fn candidate_beats(candidate_value:
|
|
38
|
+
fn candidate_beats(candidate_value: f16, candidate_index: u32, best_value: f16, best_index: u32) -> bool {
|
|
40
39
|
if (candidate_value > best_value) {
|
|
41
40
|
return true;
|
|
42
41
|
}
|
|
@@ -50,9 +49,9 @@ fn candidate_beats(candidate_value: f32, candidate_index: u32, best_value: f32,
|
|
|
50
49
|
@group(0) @binding(1) var<storage, read> logits: array<f16>;
|
|
51
50
|
@group(0) @binding(2) var<storage, read_write> output: array<u32>;
|
|
52
51
|
@group(0) @binding(3) var<storage, read_write> topk_indices: array<u32>;
|
|
53
|
-
@group(0) @binding(4) var<storage, read_write> topk_logits: array<
|
|
52
|
+
@group(0) @binding(4) var<storage, read_write> topk_logits: array<f16>;
|
|
54
53
|
|
|
55
|
-
var<workgroup> shared_values: array<
|
|
54
|
+
var<workgroup> shared_values: array<f16, MAX_WORKGROUP_SIZE>;
|
|
56
55
|
var<workgroup> shared_indices: array<u32, MAX_WORKGROUP_SIZE>;
|
|
57
56
|
|
|
58
57
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
@@ -65,27 +64,27 @@ fn find_topk_phase1(
|
|
|
65
64
|
let thread_idx = lid.x;
|
|
66
65
|
let global_idx = gid.x;
|
|
67
66
|
let vocab_size = u.vocab_size;
|
|
68
|
-
let temperature = u.temperature;
|
|
67
|
+
let temperature = f16(u.temperature);
|
|
69
68
|
let pad_id = u.pad_token_id;
|
|
70
|
-
let softcap = u.logit_softcap;
|
|
69
|
+
let softcap = f16(u.logit_softcap);
|
|
71
70
|
|
|
72
71
|
if (num_wg.x == 1u) {
|
|
73
|
-
var val:
|
|
72
|
+
var val: f16 = NEG_INF;
|
|
74
73
|
if (global_idx < vocab_size && global_idx != pad_id) {
|
|
75
|
-
val = apply_softcap(
|
|
74
|
+
val = apply_softcap(logits[global_idx], softcap) / temperature;
|
|
76
75
|
}
|
|
77
76
|
topk_logits[thread_idx] = val;
|
|
78
77
|
topk_indices[thread_idx] = global_idx;
|
|
79
78
|
return;
|
|
80
79
|
}
|
|
81
80
|
|
|
82
|
-
var local_max:
|
|
81
|
+
var local_max: f16 = NEG_INF;
|
|
83
82
|
var local_max_idx: u32 = 0u;
|
|
84
83
|
|
|
85
84
|
var idx = global_idx;
|
|
86
85
|
while (idx < vocab_size) {
|
|
87
86
|
if (idx != pad_id) {
|
|
88
|
-
let val = apply_softcap(
|
|
87
|
+
let val = apply_softcap(logits[idx], softcap) / temperature;
|
|
89
88
|
if (candidate_beats(val, idx, local_max, local_max_idx)) {
|
|
90
89
|
local_max = val;
|
|
91
90
|
local_max_idx = idx;
|
|
@@ -135,7 +134,7 @@ fn find_topk_phase2(
|
|
|
135
134
|
shared_values[thread_idx] = topk_logits[thread_idx];
|
|
136
135
|
shared_indices[thread_idx] = topk_indices[thread_idx];
|
|
137
136
|
} else {
|
|
138
|
-
shared_values[thread_idx] =
|
|
137
|
+
shared_values[thread_idx] = NEG_INF;
|
|
139
138
|
shared_indices[thread_idx] = 0u;
|
|
140
139
|
}
|
|
141
140
|
}
|
|
@@ -176,7 +175,7 @@ fn softmax_and_sample(
|
|
|
176
175
|
) {
|
|
177
176
|
let thread_idx = lid.x;
|
|
178
177
|
let top_k = u.top_k;
|
|
179
|
-
let random_val = u.random_value;
|
|
178
|
+
let random_val = f16(u.random_value);
|
|
180
179
|
|
|
181
180
|
if (thread_idx < top_k) {
|
|
182
181
|
shared_values[thread_idx] = topk_logits[thread_idx];
|
|
@@ -185,20 +184,20 @@ fn softmax_and_sample(
|
|
|
185
184
|
workgroupBarrier();
|
|
186
185
|
|
|
187
186
|
if (thread_idx == 0u) {
|
|
188
|
-
var max_val:
|
|
187
|
+
var max_val: f16 = shared_values[0];
|
|
189
188
|
for (var i: u32 = 1u; i < top_k; i = i + 1u) {
|
|
190
189
|
max_val = max(max_val, shared_values[i]);
|
|
191
190
|
}
|
|
192
191
|
|
|
193
|
-
var exp_sum:
|
|
192
|
+
var exp_sum: f16 = f16(0.0);
|
|
194
193
|
for (var i: u32 = 0u; i < top_k; i = i + 1u) {
|
|
195
194
|
let exp_val = exp(shared_values[i] - max_val);
|
|
196
195
|
shared_values[i] = exp_val;
|
|
197
196
|
exp_sum = exp_sum + exp_val;
|
|
198
197
|
}
|
|
199
198
|
|
|
200
|
-
let inv_sum = 1.0 / exp_sum;
|
|
201
|
-
var cum_prob:
|
|
199
|
+
let inv_sum = f16(1.0) / exp_sum;
|
|
200
|
+
var cum_prob: f16 = f16(0.0);
|
|
202
201
|
var selected_token: u32 = shared_indices[top_k - 1u];
|
|
203
202
|
|
|
204
203
|
for (var i: u32 = 0u; i < top_k; i = i + 1u) {
|
|
@@ -223,18 +222,17 @@ fn sample_single_pass(
|
|
|
223
222
|
let thread_idx = lid.x;
|
|
224
223
|
let vocab_size = u.vocab_size;
|
|
225
224
|
let top_k = min(u.top_k, MAX_TOP_K);
|
|
226
|
-
let temperature = u.temperature;
|
|
227
|
-
let random_val = u.random_value;
|
|
225
|
+
let temperature = f16(u.temperature);
|
|
228
226
|
let pad_id = u.pad_token_id;
|
|
229
|
-
let softcap = u.logit_softcap;
|
|
227
|
+
let softcap = f16(u.logit_softcap);
|
|
230
228
|
|
|
231
|
-
var local_max:
|
|
229
|
+
var local_max: f16 = NEG_INF;
|
|
232
230
|
var local_max_idx: u32 = 0u;
|
|
233
231
|
|
|
234
232
|
var idx = gid.x;
|
|
235
233
|
while (idx < vocab_size) {
|
|
236
234
|
if (idx != pad_id) {
|
|
237
|
-
let val = apply_softcap(
|
|
235
|
+
let val = apply_softcap(logits[idx], softcap) / temperature;
|
|
238
236
|
if (candidate_beats(val, idx, local_max, local_max_idx)) {
|
|
239
237
|
local_max = val;
|
|
240
238
|
local_max_idx = idx;
|
|
@@ -280,15 +278,15 @@ fn argmax(
|
|
|
280
278
|
let global_idx = gid.x;
|
|
281
279
|
let vocab_size = u.vocab_size;
|
|
282
280
|
let pad_id = u.pad_token_id;
|
|
283
|
-
let softcap = u.logit_softcap;
|
|
281
|
+
let softcap = f16(u.logit_softcap);
|
|
284
282
|
|
|
285
|
-
var local_max:
|
|
283
|
+
var local_max: f16 = NEG_INF;
|
|
286
284
|
var local_max_idx: u32 = 0u;
|
|
287
285
|
|
|
288
286
|
var idx = global_idx;
|
|
289
287
|
while (idx < vocab_size) {
|
|
290
288
|
if (idx != pad_id) {
|
|
291
|
-
let val = apply_softcap(
|
|
289
|
+
let val = apply_softcap(logits[idx], softcap);
|
|
292
290
|
if (candidate_beats(val, idx, local_max, local_max_idx)) {
|
|
293
291
|
local_max = val;
|
|
294
292
|
local_max_idx = idx;
|
|
@@ -335,7 +333,7 @@ fn argmax_reduce(
|
|
|
335
333
|
shared_values[thread_idx] = topk_logits[thread_idx];
|
|
336
334
|
shared_indices[thread_idx] = topk_indices[thread_idx];
|
|
337
335
|
} else {
|
|
338
|
-
shared_values[thread_idx] =
|
|
336
|
+
shared_values[thread_idx] = NEG_INF;
|
|
339
337
|
shared_indices[thread_idx] = 0u;
|
|
340
338
|
}
|
|
341
339
|
workgroupBarrier();
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/scatter_add_dynamic.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// scatter_add_dynamic_f16.wgsl
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -54,3 +54,16 @@ export declare function getShaderCacheStats(): {
|
|
|
54
54
|
sources: number;
|
|
55
55
|
modules: number;
|
|
56
56
|
};
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Register pre-fetched shader sources so the runtime does not have to fetch
|
|
60
|
+
* them over HTTP. Consumer-provided bundles call this at startup.
|
|
61
|
+
*/
|
|
62
|
+
export declare function registerShaderSources(
|
|
63
|
+
sources: Record<string, string> | Map<string, string>
|
|
64
|
+
): void;
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Report whether a pre-seeded source is registered for a given shader file.
|
|
68
|
+
*/
|
|
69
|
+
export declare function hasPreseededShaderSource(shaderFile: string): boolean;
|
|
@@ -111,12 +111,39 @@ async function loadShaderSourceFromFileUrl(url) {
|
|
|
111
111
|
// ============================================================================
|
|
112
112
|
|
|
113
113
|
|
|
114
|
+
// Consumer-preseeded shader sources. Bundlers that can resolve
|
|
115
|
+
// `import.meta.glob('./kernels/*.wgsl', { as: 'raw', eager: true })` or
|
|
116
|
+
// equivalent should call `registerShaderSources(map)` before any pipeline
|
|
117
|
+
// init; the runtime skips the fetch path entirely when a preseed is present.
|
|
118
|
+
const shaderSourcePreseeds = new Map();
|
|
119
|
+
|
|
120
|
+
export function registerShaderSources(map) {
|
|
121
|
+
if (!map) return;
|
|
122
|
+
const entries = map instanceof Map ? map.entries() : Object.entries(map);
|
|
123
|
+
for (const [key, value] of entries) {
|
|
124
|
+
if (typeof value !== 'string') continue;
|
|
125
|
+
const filename = String(key).split('/').pop();
|
|
126
|
+
if (!filename) continue;
|
|
127
|
+
shaderSourcePreseeds.set(filename, value);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export function hasPreseededShaderSource(filename) {
|
|
132
|
+
return shaderSourcePreseeds.has(filename);
|
|
133
|
+
}
|
|
134
|
+
|
|
114
135
|
export async function loadShaderSource(filename) {
|
|
115
136
|
if (shaderSourceCache.has(filename)) {
|
|
116
137
|
const cached = shaderSourceCache.get(filename);
|
|
117
138
|
touchCacheEntry(shaderSourceCache, filename, cached);
|
|
118
139
|
return cached;
|
|
119
140
|
}
|
|
141
|
+
const preseeded = shaderSourcePreseeds.get(filename);
|
|
142
|
+
if (typeof preseeded === 'string') {
|
|
143
|
+
shaderSourceCache.set(filename, preseeded);
|
|
144
|
+
evictOldest(shaderSourceCache, MAX_SHADER_SOURCE_CACHE_SIZE);
|
|
145
|
+
return preseeded;
|
|
146
|
+
}
|
|
120
147
|
|
|
121
148
|
const url = `${KERNEL_BASE_PATH}/${filename}`;
|
|
122
149
|
try {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/silu.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
//
|
|
4
4
|
// F16 variant for reduced memory bandwidth when using F16 activations.
|
|
5
5
|
// Intermediate computations use F32 for precision.
|