doppler-gpu 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +19 -15
- package/package.json +77 -29
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +64 -6
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +524 -350
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +12 -2
- package/src/config/schema/doppler.schema.js +108 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +68 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +76 -0
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +107 -0
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +3 -21
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +6 -5
- package/src/gpu/kernels/index.js +3 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +12 -0
- package/src/inference/browser-harness-text-helpers.js +223 -14
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +264 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +22 -27
- package/src/inference/pipelines/text/embed.js +0 -39
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +4 -5
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +8 -3
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +223 -547
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +5 -2
- package/src/inference/pipelines/text/init.js +183 -724
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.js +2 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +120 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +223 -27
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +13 -3
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +11 -3
- package/src/storage/artifact-storage-context.js +231 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +21 -24
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +2 -0
- package/src/tooling/source-runtime-bundle.js +3 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -12,6 +12,17 @@ import { unifiedKernelWrapper } from './utils.js';
|
|
|
12
12
|
// Conservative fallback dtype for norm weight inference when metadata is unavailable.
|
|
13
13
|
const DEFAULT_DTYPE = 'f32';
|
|
14
14
|
|
|
15
|
+
// Must equal MAX_CACHE_SIZE in rmsnorm.wgsl. The "residual" variant routes to
|
|
16
|
+
// main_cached, which caches the input in shared_cache[MAX_CACHE_SIZE]. When
|
|
17
|
+
// hiddenSize exceeds this limit, OOB cache writes silently drop indices and the
|
|
18
|
+
// second pass reads zero for those dims, producing wrong output (only residual
|
|
19
|
+
// preserved). Models with hiddenSize > limit must skip the cached variant.
|
|
20
|
+
export const RMSNORM_CACHE_LIMIT = 4608;
|
|
21
|
+
|
|
22
|
+
export function residualVariantBypassesCache(residual, hiddenSize) {
|
|
23
|
+
return !!residual && hiddenSize !== null && hiddenSize !== undefined && hiddenSize > RMSNORM_CACHE_LIMIT;
|
|
24
|
+
}
|
|
25
|
+
|
|
15
26
|
function inferHiddenSize(input, hiddenSize) {
|
|
16
27
|
if (hiddenSize != null) return hiddenSize;
|
|
17
28
|
const shape = input?.shape;
|
|
@@ -104,7 +115,12 @@ export function selectRMSNormKernel(options = {}, isF16 = false) {
|
|
|
104
115
|
return selectRuleValue(
|
|
105
116
|
'rmsnorm',
|
|
106
117
|
'variant',
|
|
107
|
-
{
|
|
118
|
+
{
|
|
119
|
+
isF16,
|
|
120
|
+
residual: !!residual && !residualVariantBypassesCache(residual, hiddenSize),
|
|
121
|
+
hasSubgroups,
|
|
122
|
+
isSmall,
|
|
123
|
+
}
|
|
108
124
|
);
|
|
109
125
|
}
|
|
110
126
|
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
//
|
|
11
11
|
// Uses override constants for compile-time feature selection:
|
|
12
12
|
// - RMS_NORM_OFFSET: Use (1 + weight) pattern (Gemma models)
|
|
13
|
-
// - HAS_RESIDUAL:
|
|
13
|
+
// - HAS_RESIDUAL: Compile-time override (reserved for codegen patches, not used by JS dispatch)
|
|
14
14
|
// - OUTPUT_PRENORM: Output pre-normalized values (before weight multiplication)
|
|
15
15
|
|
|
16
16
|
enable subgroups;
|
|
@@ -30,6 +30,7 @@ override OUTPUT_PRENORM: bool = false; // Write pre-norm sum to residual_sum_
|
|
|
30
30
|
override WEIGHT_IS_F16: bool = false; // Weight buffer packed as f16 pairs
|
|
31
31
|
|
|
32
32
|
const MAX_SUBGROUPS: u32 = 32u; // Support up to 32 subgroups per workgroup
|
|
33
|
+
const MAX_CACHE_SIZE: u32 = 4608u; // Supports hiddenSize up to 1152 with 4 elements/thread
|
|
33
34
|
|
|
34
35
|
// =============================================================================
|
|
35
36
|
// Uniforms (per-dispatch)
|
|
@@ -81,11 +82,9 @@ fn load_weight(idx: u32) -> f32 {
|
|
|
81
82
|
return bitcast<f32>(weight[idx]);
|
|
82
83
|
}
|
|
83
84
|
|
|
84
|
-
// Check if residual should be added (
|
|
85
|
+
// Check if residual should be added (runtime uniform only — JS dispatch controls this)
|
|
85
86
|
fn should_add_residual() -> bool {
|
|
86
|
-
|
|
87
|
-
// u.has_residual is runtime flag (for flexibility)
|
|
88
|
-
return HAS_RESIDUAL || (u.has_residual != 0u);
|
|
87
|
+
return u.has_residual != 0u;
|
|
89
88
|
}
|
|
90
89
|
|
|
91
90
|
// Load input value, fusing residual add when PRE_RESIDUAL is active
|
|
@@ -242,7 +241,7 @@ fn main_small(
|
|
|
242
241
|
|
|
243
242
|
// OPTIMIZED: Caches input to avoid double loads
|
|
244
243
|
// Uses shared memory to store input values between passes
|
|
245
|
-
var<workgroup> shared_cache: array<f32,
|
|
244
|
+
var<workgroup> shared_cache: array<f32, MAX_CACHE_SIZE>;
|
|
246
245
|
|
|
247
246
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
248
247
|
fn main_cached(
|
|
@@ -1,12 +1,9 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
3
|
-
// AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
|
|
4
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
|
|
5
|
-
// RMSNorm Kernel with F16 Input/Output
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
6
3
|
//
|
|
7
4
|
// F16 variant for reduced memory bandwidth when using F16 activations.
|
|
8
|
-
//
|
|
9
|
-
// Weight buffer may be F16 or F32
|
|
5
|
+
// The all-f16 lane keeps normalization arithmetic in f16.
|
|
6
|
+
// Weight buffer may be F16 or F32.
|
|
10
7
|
//
|
|
11
8
|
// RMSNorm(x) = x / sqrt(mean(x^2) + eps) * weight
|
|
12
9
|
|
|
@@ -37,38 +34,37 @@ struct Uniforms {
|
|
|
37
34
|
@group(0) @binding(4) var<storage, read> residual: array<f16>; // Optional residual
|
|
38
35
|
@group(0) @binding(5) var<storage, read_write> residual_sum_output: array<f16>; // Pre-norm sum output (PRE_RESIDUAL + OUTPUT_PRENORM)
|
|
39
36
|
|
|
40
|
-
|
|
41
|
-
var<workgroup> shared_sum: array<f32, MAX_WORKGROUP_SIZE>;
|
|
37
|
+
var<workgroup> shared_sum: array<f16, MAX_WORKGROUP_SIZE>;
|
|
42
38
|
|
|
43
|
-
fn apply_weight(w:
|
|
39
|
+
fn apply_weight(w: f16) -> f16 {
|
|
44
40
|
if (RMS_NORM_OFFSET) {
|
|
45
|
-
return 1.0 + w;
|
|
41
|
+
return f16(1.0) + w;
|
|
46
42
|
}
|
|
47
43
|
return w;
|
|
48
44
|
}
|
|
49
45
|
|
|
50
|
-
fn load_weight(idx: u32) ->
|
|
46
|
+
fn load_weight(idx: u32) -> f16 {
|
|
51
47
|
if (WEIGHT_IS_F16) {
|
|
52
48
|
let packed = weight[idx >> 1u];
|
|
53
49
|
let pair = unpack2x16float(packed);
|
|
54
|
-
return select(pair.x, pair.y, (idx & 1u) == 1u);
|
|
50
|
+
return f16(select(pair.x, pair.y, (idx & 1u) == 1u));
|
|
55
51
|
}
|
|
56
|
-
return bitcast<f32>(weight[idx]);
|
|
52
|
+
return f16(bitcast<f32>(weight[idx]));
|
|
57
53
|
}
|
|
58
54
|
|
|
59
55
|
// Load input value, fusing residual add when PRE_RESIDUAL is active
|
|
60
|
-
fn load_input(base_offset: u32, idx: u32) ->
|
|
61
|
-
let x =
|
|
56
|
+
fn load_input(base_offset: u32, idx: u32) -> f16 {
|
|
57
|
+
let x = input[base_offset + idx];
|
|
62
58
|
if (PRE_RESIDUAL) {
|
|
63
|
-
return x +
|
|
59
|
+
return x + residual[base_offset + idx];
|
|
64
60
|
}
|
|
65
61
|
return x;
|
|
66
62
|
}
|
|
67
63
|
|
|
68
64
|
// Write pre-norm sum when OUTPUT_PRENORM is active (for downstream residual reuse)
|
|
69
|
-
fn write_prenorm(base_offset: u32, idx: u32, val:
|
|
65
|
+
fn write_prenorm(base_offset: u32, idx: u32, val: f16) {
|
|
70
66
|
if (OUTPUT_PRENORM && PRE_RESIDUAL) {
|
|
71
|
-
residual_sum_output[base_offset + idx] =
|
|
67
|
+
residual_sum_output[base_offset + idx] = val;
|
|
72
68
|
}
|
|
73
69
|
}
|
|
74
70
|
|
|
@@ -93,16 +89,36 @@ fn main(
|
|
|
93
89
|
|
|
94
90
|
let base_offset = token_idx * size;
|
|
95
91
|
|
|
96
|
-
// Each thread computes partial sum of squares (promote to F32 for precision)
|
|
97
|
-
// When PRE_RESIDUAL is active, sum is computed on (input + residual)
|
|
98
|
-
var local_sum_sq: f32 = 0.0;
|
|
99
92
|
let elements_per_thread = (size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE;
|
|
100
93
|
|
|
94
|
+
var local_max: f16 = f16(0.0);
|
|
101
95
|
for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
|
|
102
96
|
let idx = thread_idx * elements_per_thread + i;
|
|
103
97
|
if (idx < size) {
|
|
104
|
-
|
|
105
|
-
|
|
98
|
+
local_max = max(local_max, abs(load_input(base_offset, idx)));
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
shared_sum[thread_idx] = local_max;
|
|
103
|
+
workgroupBarrier();
|
|
104
|
+
|
|
105
|
+
for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
|
|
106
|
+
if (thread_idx < stride) {
|
|
107
|
+
shared_sum[thread_idx] = max(shared_sum[thread_idx], shared_sum[thread_idx + stride]);
|
|
108
|
+
}
|
|
109
|
+
workgroupBarrier();
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
let row_scale = select(f16(1.0), shared_sum[0], shared_sum[0] > f16(0.0));
|
|
113
|
+
workgroupBarrier();
|
|
114
|
+
|
|
115
|
+
var local_sum_sq: f16 = f16(0.0);
|
|
116
|
+
let inv_size = f16(1.0) / f16(size);
|
|
117
|
+
for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
|
|
118
|
+
let idx = thread_idx * elements_per_thread + i;
|
|
119
|
+
if (idx < size) {
|
|
120
|
+
let x = load_input(base_offset, idx) / row_scale;
|
|
121
|
+
local_sum_sq = local_sum_sq + (x * x) * inv_size;
|
|
106
122
|
}
|
|
107
123
|
}
|
|
108
124
|
|
|
@@ -118,10 +134,9 @@ fn main(
|
|
|
118
134
|
workgroupBarrier();
|
|
119
135
|
}
|
|
120
136
|
|
|
121
|
-
|
|
122
|
-
let
|
|
123
|
-
let
|
|
124
|
-
let inv_rms = 1.0 / rms;
|
|
137
|
+
let mean_sq = shared_sum[0];
|
|
138
|
+
let eps_scaled = f16(u.eps) / (row_scale * row_scale);
|
|
139
|
+
let inv_rms = (f16(1.0) / row_scale) / sqrt(mean_sq + eps_scaled);
|
|
125
140
|
|
|
126
141
|
workgroupBarrier();
|
|
127
142
|
|
|
@@ -131,19 +146,17 @@ fn main(
|
|
|
131
146
|
if (idx < size) {
|
|
132
147
|
let x = load_input(base_offset, idx);
|
|
133
148
|
|
|
134
|
-
// Normalize and scale (compute in F32)
|
|
135
149
|
var result = x * inv_rms * apply_weight(load_weight(idx));
|
|
136
150
|
|
|
137
151
|
// Add residual AFTER normalization
|
|
138
152
|
if (u.has_residual == 1u) {
|
|
139
|
-
result = result +
|
|
153
|
+
result = result + residual[base_offset + idx];
|
|
140
154
|
}
|
|
141
155
|
|
|
142
156
|
// Write pre-norm sum for downstream residual reuse
|
|
143
157
|
write_prenorm(base_offset, idx, x);
|
|
144
158
|
|
|
145
|
-
|
|
146
|
-
output[base_offset + idx] = f16(result);
|
|
159
|
+
output[base_offset + idx] = result;
|
|
147
160
|
}
|
|
148
161
|
}
|
|
149
162
|
}
|
|
@@ -165,13 +178,25 @@ fn rmsnorm_small_f16(
|
|
|
165
178
|
let base_offset = token_idx * size;
|
|
166
179
|
|
|
167
180
|
// Each thread handles one element (for size <= 256)
|
|
168
|
-
var x:
|
|
181
|
+
var x: f16 = f16(0.0);
|
|
169
182
|
if (thread_idx < size) {
|
|
170
183
|
x = load_input(base_offset, thread_idx);
|
|
171
184
|
}
|
|
172
185
|
|
|
173
|
-
|
|
174
|
-
|
|
186
|
+
shared_sum[thread_idx] = abs(x);
|
|
187
|
+
workgroupBarrier();
|
|
188
|
+
|
|
189
|
+
for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
|
|
190
|
+
if (thread_idx < stride) {
|
|
191
|
+
shared_sum[thread_idx] = max(shared_sum[thread_idx], shared_sum[thread_idx + stride]);
|
|
192
|
+
}
|
|
193
|
+
workgroupBarrier();
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
let row_scale = select(f16(1.0), shared_sum[0], shared_sum[0] > f16(0.0));
|
|
197
|
+
let x_scaled = x / row_scale;
|
|
198
|
+
let inv_size = f16(1.0) / f16(size);
|
|
199
|
+
shared_sum[thread_idx] = (x_scaled * x_scaled) * inv_size;
|
|
175
200
|
workgroupBarrier();
|
|
176
201
|
|
|
177
202
|
// Parallel reduction
|
|
@@ -183,16 +208,17 @@ fn rmsnorm_small_f16(
|
|
|
183
208
|
}
|
|
184
209
|
|
|
185
210
|
// Compute inverse RMS
|
|
186
|
-
let mean_sq = shared_sum[0]
|
|
187
|
-
let
|
|
211
|
+
let mean_sq = shared_sum[0];
|
|
212
|
+
let eps_scaled = f16(u.eps) / (row_scale * row_scale);
|
|
213
|
+
let inv_rms = (f16(1.0) / row_scale) / sqrt(mean_sq + eps_scaled);
|
|
188
214
|
|
|
189
|
-
// Apply normalization
|
|
215
|
+
// Apply normalization
|
|
190
216
|
if (thread_idx < size) {
|
|
191
217
|
var result = x * inv_rms * apply_weight(load_weight(thread_idx));
|
|
192
218
|
if (u.has_residual == 1u) {
|
|
193
|
-
result = result +
|
|
219
|
+
result = result + residual[base_offset + thread_idx];
|
|
194
220
|
}
|
|
195
221
|
write_prenorm(base_offset, thread_idx, x);
|
|
196
|
-
output[base_offset + thread_idx] =
|
|
222
|
+
output[base_offset + thread_idx] = result;
|
|
197
223
|
}
|
|
198
224
|
}
|
|
@@ -17,17 +17,13 @@ export interface RoPEOptions extends OutputBufferOptions {
|
|
|
17
17
|
numHeads?: number;
|
|
18
18
|
headDim?: number;
|
|
19
19
|
rotaryDim?: number;
|
|
20
|
+
pairSpanDim?: number;
|
|
20
21
|
interleaved?: boolean;
|
|
21
22
|
ropeTheta?: number;
|
|
22
23
|
startPos?: number;
|
|
23
24
|
executionPolicies?: ExecutionV1PoliciesSchema | null;
|
|
24
25
|
}
|
|
25
26
|
|
|
26
|
-
/** Context for RoPE variant selection rules. */
|
|
27
|
-
export interface RoPESelectionContext {
|
|
28
|
-
useF16: boolean;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
27
|
/**
|
|
32
28
|
* Run RoPE operation
|
|
33
29
|
*/
|
package/src/gpu/kernels/rope.js
CHANGED
|
@@ -17,19 +17,28 @@ async function _rope(target, input, freqsCos, freqsSin, seqLen, options = {}) {
|
|
|
17
17
|
numHeads = 1,
|
|
18
18
|
headDim = 64,
|
|
19
19
|
rotaryDim = headDim,
|
|
20
|
+
pairSpanDim = rotaryDim,
|
|
20
21
|
interleaved = false,
|
|
21
22
|
ropeTheta = ropeDefaults.defaultTheta,
|
|
22
23
|
} = options;
|
|
23
24
|
|
|
24
|
-
if (headDim % 2 !== 0) {
|
|
25
|
+
if (!Number.isFinite(headDim) || headDim % 2 !== 0) {
|
|
25
26
|
throw new Error(`RoPE headDim must be even, got ${headDim}`);
|
|
26
27
|
}
|
|
27
|
-
if (rotaryDim % 2 !== 0) {
|
|
28
|
+
if (!Number.isFinite(rotaryDim) || rotaryDim % 2 !== 0) {
|
|
28
29
|
throw new Error(`RoPE rotaryDim must be even, got ${rotaryDim}`);
|
|
29
30
|
}
|
|
30
31
|
if (rotaryDim <= 0 || rotaryDim > headDim) {
|
|
31
32
|
throw new Error(`RoPE rotaryDim must be in (0, headDim]; got ${rotaryDim} for headDim ${headDim}`);
|
|
32
33
|
}
|
|
34
|
+
if (!Number.isFinite(pairSpanDim) || pairSpanDim % 2 !== 0) {
|
|
35
|
+
throw new Error(`RoPE pairSpanDim must be even, got ${pairSpanDim}`);
|
|
36
|
+
}
|
|
37
|
+
if (pairSpanDim < rotaryDim || pairSpanDim > headDim) {
|
|
38
|
+
throw new Error(
|
|
39
|
+
`RoPE pairSpanDim must be in [rotaryDim, headDim]; got ${pairSpanDim} for rotaryDim ${rotaryDim} and headDim ${headDim}`
|
|
40
|
+
);
|
|
41
|
+
}
|
|
33
42
|
|
|
34
43
|
const caps = getKernelCapabilities();
|
|
35
44
|
const needsF32Cast = input.dtype === 'f16'
|
|
@@ -70,6 +79,7 @@ async function _rope(target, input, freqsCos, freqsSin, seqLen, options = {}) {
|
|
|
70
79
|
rope_base: ropeTheta,
|
|
71
80
|
rope_scale: 1.0,
|
|
72
81
|
interleaved: interleaved ? 1 : 0,
|
|
82
|
+
pair_span_dim: pairSpanDim,
|
|
73
83
|
},
|
|
74
84
|
workgroups
|
|
75
85
|
);
|
|
@@ -16,6 +16,11 @@
|
|
|
16
16
|
|
|
17
17
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
18
18
|
|
|
19
|
+
// YaRN parameters (model-family-specific, config-driven)
|
|
20
|
+
override YARN_BETA_FAST: f32 = 32.0;
|
|
21
|
+
override YARN_BETA_SLOW: f32 = 1.0;
|
|
22
|
+
override YARN_ALPHA: f32 = 1.0;
|
|
23
|
+
|
|
19
24
|
// Mathematical constant
|
|
20
25
|
const PI: f32 = 3.14159265359;
|
|
21
26
|
|
|
@@ -28,6 +33,10 @@ struct Uniforms {
|
|
|
28
33
|
rope_scale: f32, // Scaling factor for extended context
|
|
29
34
|
rotary_dim: u32, // Rotary slice within head_dim
|
|
30
35
|
interleaved: u32, // 1 = adjacent pairs, 0 = rotate-half
|
|
36
|
+
pair_span_dim: u32, // Rotate-half partner span within head_dim
|
|
37
|
+
_pad0: u32,
|
|
38
|
+
_pad1: u32,
|
|
39
|
+
_pad2: u32,
|
|
31
40
|
}
|
|
32
41
|
|
|
33
42
|
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
@@ -42,11 +51,11 @@ fn get_first_rotary_idx(pair_idx: u32) -> u32 {
|
|
|
42
51
|
return pair_idx;
|
|
43
52
|
}
|
|
44
53
|
|
|
45
|
-
fn get_second_rotary_idx(pair_idx: u32,
|
|
54
|
+
fn get_second_rotary_idx(pair_idx: u32, pair_span_dim: u32) -> u32 {
|
|
46
55
|
if (u.interleaved == 1u) {
|
|
47
56
|
return pair_idx * 2u + 1u;
|
|
48
57
|
}
|
|
49
|
-
return pair_idx + (
|
|
58
|
+
return pair_idx + (pair_span_dim / 2u);
|
|
50
59
|
}
|
|
51
60
|
|
|
52
61
|
// Apply RoPE using precomputed frequencies
|
|
@@ -83,7 +92,7 @@ fn main(
|
|
|
83
92
|
|
|
84
93
|
let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
|
|
85
94
|
let first_idx = get_first_rotary_idx(pair_idx);
|
|
86
|
-
let second_idx = get_second_rotary_idx(pair_idx,
|
|
95
|
+
let second_idx = get_second_rotary_idx(pair_idx, u.pair_span_dim);
|
|
87
96
|
let x0 = input[base_idx + first_idx];
|
|
88
97
|
let x1 = input[base_idx + second_idx];
|
|
89
98
|
|
|
@@ -108,6 +117,7 @@ fn rope_compute_freqs(
|
|
|
108
117
|
let rope_base = u.rope_base;
|
|
109
118
|
let rope_scale = u.rope_scale;
|
|
110
119
|
let rotary_dim = u.rotary_dim;
|
|
120
|
+
let pair_span_dim = u.pair_span_dim;
|
|
111
121
|
|
|
112
122
|
let idx = global_id.x;
|
|
113
123
|
let half_dim = rotary_dim / 2u;
|
|
@@ -126,7 +136,7 @@ fn rope_compute_freqs(
|
|
|
126
136
|
let actual_pos = f32(start_pos + pos) / rope_scale;
|
|
127
137
|
|
|
128
138
|
// Compute frequency: 1 / (base^(2*pair_idx/head_dim))
|
|
129
|
-
let exponent = f32(pair_idx * 2u) / f32(
|
|
139
|
+
let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
|
|
130
140
|
let freq = 1.0 / pow(rope_base, exponent);
|
|
131
141
|
let theta = actual_pos * freq;
|
|
132
142
|
|
|
@@ -135,7 +145,7 @@ fn rope_compute_freqs(
|
|
|
135
145
|
|
|
136
146
|
let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
|
|
137
147
|
let first_idx = get_first_rotary_idx(pair_idx);
|
|
138
|
-
let second_idx = get_second_rotary_idx(pair_idx,
|
|
148
|
+
let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
|
|
139
149
|
let x0 = input[base_idx + first_idx];
|
|
140
150
|
let x1 = input[base_idx + second_idx];
|
|
141
151
|
|
|
@@ -157,6 +167,7 @@ fn rope_qk(
|
|
|
157
167
|
let rope_base = u.rope_base;
|
|
158
168
|
let rope_scale = u.rope_scale;
|
|
159
169
|
let rotary_dim = u.rotary_dim;
|
|
170
|
+
let pair_span_dim = u.pair_span_dim;
|
|
160
171
|
|
|
161
172
|
let idx = global_id.x;
|
|
162
173
|
// Each thread handles one Q-K pair at one dimension pair
|
|
@@ -175,7 +186,7 @@ fn rope_qk(
|
|
|
175
186
|
let actual_pos = f32(start_pos + pos) / rope_scale;
|
|
176
187
|
|
|
177
188
|
// Compute frequency
|
|
178
|
-
let exponent = f32(pair_idx * 2u) / f32(
|
|
189
|
+
let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
|
|
179
190
|
let freq = 1.0 / pow(rope_base, exponent);
|
|
180
191
|
let theta = actual_pos * freq;
|
|
181
192
|
|
|
@@ -188,7 +199,7 @@ fn rope_qk(
|
|
|
188
199
|
|
|
189
200
|
// Process Q
|
|
190
201
|
let first_idx = get_first_rotary_idx(pair_idx);
|
|
191
|
-
let second_idx = get_second_rotary_idx(pair_idx,
|
|
202
|
+
let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
|
|
192
203
|
let q0 = input[q_base_idx + first_idx];
|
|
193
204
|
let q1 = input[q_base_idx + second_idx];
|
|
194
205
|
input[q_base_idx + first_idx] = q0 * cos_val - q1 * sin_val;
|
|
@@ -212,6 +223,7 @@ fn precompute_freqs(
|
|
|
212
223
|
let rope_base = u.rope_base;
|
|
213
224
|
let rope_scale = u.rope_scale;
|
|
214
225
|
let rotary_dim = u.rotary_dim;
|
|
226
|
+
let pair_span_dim = u.pair_span_dim;
|
|
215
227
|
|
|
216
228
|
let idx = global_id.x;
|
|
217
229
|
let half_dim = rotary_dim / 2u;
|
|
@@ -225,7 +237,7 @@ fn precompute_freqs(
|
|
|
225
237
|
let dim_idx = idx % half_dim;
|
|
226
238
|
|
|
227
239
|
let actual_pos = f32(pos) / rope_scale;
|
|
228
|
-
let exponent = f32(dim_idx * 2u) / f32(
|
|
240
|
+
let exponent = f32(dim_idx * 2u) / f32(pair_span_dim);
|
|
229
241
|
let freq = 1.0 / pow(rope_base, exponent);
|
|
230
242
|
let theta = actual_pos * freq;
|
|
231
243
|
|
|
@@ -246,6 +258,7 @@ fn rope_ntk_scaled(
|
|
|
246
258
|
let start_pos = u.start_pos;
|
|
247
259
|
var rope_base = u.rope_base;
|
|
248
260
|
let rope_scale = u.rope_scale;
|
|
261
|
+
let pair_span_dim = u.pair_span_dim;
|
|
249
262
|
|
|
250
263
|
let idx = global_id.x;
|
|
251
264
|
let half_dim = rotary_dim / 2u;
|
|
@@ -257,7 +270,7 @@ fn rope_ntk_scaled(
|
|
|
257
270
|
|
|
258
271
|
// NTK scaling: increase base proportionally to scale factor
|
|
259
272
|
// This preserves high-frequency components better than linear interpolation
|
|
260
|
-
rope_base = rope_base * pow(rope_scale, f32(
|
|
273
|
+
rope_base = rope_base * pow(rope_scale, f32(pair_span_dim) / (f32(pair_span_dim) - 2.0));
|
|
261
274
|
|
|
262
275
|
let pos = idx / (num_heads * half_dim);
|
|
263
276
|
let remainder = idx % (num_heads * half_dim);
|
|
@@ -266,7 +279,7 @@ fn rope_ntk_scaled(
|
|
|
266
279
|
|
|
267
280
|
let actual_pos = f32(start_pos + pos);
|
|
268
281
|
|
|
269
|
-
let exponent = f32(pair_idx * 2u) / f32(
|
|
282
|
+
let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
|
|
270
283
|
let freq = 1.0 / pow(rope_base, exponent);
|
|
271
284
|
let theta = actual_pos * freq;
|
|
272
285
|
|
|
@@ -275,7 +288,7 @@ fn rope_ntk_scaled(
|
|
|
275
288
|
|
|
276
289
|
let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
|
|
277
290
|
let first_idx = get_first_rotary_idx(pair_idx);
|
|
278
|
-
let second_idx = get_second_rotary_idx(pair_idx,
|
|
291
|
+
let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
|
|
279
292
|
let x0 = input[base_idx + first_idx];
|
|
280
293
|
let x1 = input[base_idx + second_idx];
|
|
281
294
|
|
|
@@ -296,6 +309,7 @@ fn rope_yarn(
|
|
|
296
309
|
let start_pos = u.start_pos;
|
|
297
310
|
let rope_base = u.rope_base;
|
|
298
311
|
let rope_scale = u.rope_scale;
|
|
312
|
+
let pair_span_dim = u.pair_span_dim;
|
|
299
313
|
|
|
300
314
|
let idx = global_id.x;
|
|
301
315
|
let half_dim = rotary_dim / 2u;
|
|
@@ -312,13 +326,13 @@ fn rope_yarn(
|
|
|
312
326
|
|
|
313
327
|
let actual_pos = f32(start_pos + pos);
|
|
314
328
|
|
|
315
|
-
// YaRN parameters
|
|
316
|
-
let beta_fast: f32 =
|
|
317
|
-
let beta_slow: f32 =
|
|
318
|
-
let alpha: f32 =
|
|
329
|
+
// YaRN parameters (from override constants)
|
|
330
|
+
let beta_fast: f32 = YARN_BETA_FAST;
|
|
331
|
+
let beta_slow: f32 = YARN_BETA_SLOW;
|
|
332
|
+
let alpha: f32 = YARN_ALPHA;
|
|
319
333
|
|
|
320
334
|
// Compute original frequency
|
|
321
|
-
let exponent = f32(pair_idx * 2u) / f32(
|
|
335
|
+
let exponent = f32(pair_idx * 2u) / f32(pair_span_dim);
|
|
322
336
|
let orig_freq = 1.0 / pow(rope_base, exponent);
|
|
323
337
|
|
|
324
338
|
// Compute wavelength
|
|
@@ -326,8 +340,8 @@ fn rope_yarn(
|
|
|
326
340
|
|
|
327
341
|
// Interpolation factor based on wavelength
|
|
328
342
|
var ramp: f32;
|
|
329
|
-
let low_wavelength = f32(
|
|
330
|
-
let high_wavelength = f32(
|
|
343
|
+
let low_wavelength = f32(pair_span_dim) / beta_fast;
|
|
344
|
+
let high_wavelength = f32(pair_span_dim) / beta_slow;
|
|
331
345
|
|
|
332
346
|
if (wavelength < low_wavelength) {
|
|
333
347
|
ramp = 0.0; // No interpolation for high frequencies
|
|
@@ -347,7 +361,7 @@ fn rope_yarn(
|
|
|
347
361
|
|
|
348
362
|
let base_idx = pos * num_heads * head_dim + head_idx * head_dim;
|
|
349
363
|
let first_idx = get_first_rotary_idx(pair_idx);
|
|
350
|
-
let second_idx = get_second_rotary_idx(pair_idx,
|
|
364
|
+
let second_idx = get_second_rotary_idx(pair_idx, pair_span_dim);
|
|
351
365
|
let x0 = input[base_idx + first_idx];
|
|
352
366
|
let x1 = input[base_idx + second_idx];
|
|
353
367
|
|