doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
applyAttentionValueNorm,
|
|
32
32
|
} from './projections.js';
|
|
33
33
|
import { prepareAttentionProjectionInput } from './output-projection.js';
|
|
34
|
+
import { runProbes } from '../probes.js';
|
|
34
35
|
|
|
35
36
|
import { releaseOrTrack, shouldDebugLayer } from './types.js';
|
|
36
37
|
import {
|
|
@@ -47,10 +48,12 @@ import {
|
|
|
47
48
|
buildContiguousQuantAttentionOptions,
|
|
48
49
|
} from './quant-options.js';
|
|
49
50
|
import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
|
|
51
|
+
import { getRuntimeConfig } from '../../../../config/runtime.js';
|
|
50
52
|
import {
|
|
51
53
|
resolveAttentionPrecisionContract,
|
|
52
54
|
isAttentionKvDtypeExplicit,
|
|
53
55
|
} from './precision-contract.js';
|
|
56
|
+
import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
|
|
54
57
|
|
|
55
58
|
const ATTENTION_DTYPE_LOGGED = new Set();
|
|
56
59
|
|
|
@@ -135,6 +138,7 @@ export async function recordLayerAttentionGPU(
|
|
|
135
138
|
let finalOutput = null;
|
|
136
139
|
let oProjInputTemp = null;
|
|
137
140
|
let retainSharedKvBuffers = false;
|
|
141
|
+
let valueAliasesKey = false;
|
|
138
142
|
if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
|
|
139
143
|
assertAttentionDtypeTransitionAllowed(
|
|
140
144
|
state,
|
|
@@ -160,9 +164,37 @@ export async function recordLayerAttentionGPU(
|
|
|
160
164
|
const kvSize = numTokens * numKVHeads * headDim;
|
|
161
165
|
|
|
162
166
|
// 1. Input norm
|
|
167
|
+
// Opt-in fusion: when useFusedRmsnormWideTile is set, defer the standalone
|
|
168
|
+
// rmsnorm into each q/k/v_proj matmul (fused kernel runs norm internally).
|
|
169
|
+
let fusedNormWeightRec = null;
|
|
170
|
+
let fusedNormEpsRec = null;
|
|
171
|
+
let fusedNormOffsetRec = false;
|
|
172
|
+
let fusedNormOwnedRec = false;
|
|
173
|
+
// state.runtimeConfig is a stale snapshot whose inference.session is an empty
|
|
174
|
+
// object; the live module-level runtime config carries the merged
|
|
175
|
+
// profile/override session. Read session-level flags from getRuntimeConfig().
|
|
176
|
+
const rmsNormFusionFlagRec = getRuntimeConfig()?.inference?.session?.useFusedRmsnormWideTile === true;
|
|
177
|
+
const canSelectFusedRmsNormProjectionRec = canUseRmsNormWideTileProjectionFusion(
|
|
178
|
+
layerWeights,
|
|
179
|
+
sharedKVSourceLayerIdx != null
|
|
180
|
+
);
|
|
181
|
+
const canFuseInputNormProjRec = rmsNormFusionFlagRec
|
|
182
|
+
&& canSelectFusedRmsNormProjectionRec
|
|
183
|
+
&& !skipInputNorm
|
|
184
|
+
&& layerWeights.inputNorm
|
|
185
|
+
&& getNormWeightBuffer
|
|
186
|
+
&& isPrefill
|
|
187
|
+
&& numTokens > 1
|
|
188
|
+
&& attentionInput.dtype === 'f32';
|
|
163
189
|
|
|
164
190
|
try {
|
|
165
|
-
if (
|
|
191
|
+
if (canFuseInputNormProjRec) {
|
|
192
|
+
fusedNormWeightRec = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
193
|
+
fusedNormEpsRec = rmsNormEps;
|
|
194
|
+
fusedNormOffsetRec = config.rmsNormWeightOffset === true;
|
|
195
|
+
fusedNormOwnedRec = !isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm);
|
|
196
|
+
// Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs norm internally.
|
|
197
|
+
} else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
|
|
166
198
|
const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
167
199
|
normed = await recordRMSNorm(recorder, attentionInput, normWeightBuf, rmsNormEps, {
|
|
168
200
|
batchSize: numTokens,
|
|
@@ -170,6 +202,11 @@ export async function recordLayerAttentionGPU(
|
|
|
170
202
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
171
203
|
});
|
|
172
204
|
if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseOrTrack(recorder, normWeightBuf);
|
|
205
|
+
await runProbes('post_input_norm', normed.buffer, {
|
|
206
|
+
layerIdx, numTokens, hiddenSize,
|
|
207
|
+
probes: state.debugProbes, recorder,
|
|
208
|
+
operatorDiagnostics: state.operatorDiagnostics, dtype: normed.dtype,
|
|
209
|
+
});
|
|
173
210
|
}
|
|
174
211
|
|
|
175
212
|
const debugLayers = debugFlags.debugLayers;
|
|
@@ -217,7 +254,7 @@ export async function recordLayerAttentionGPU(
|
|
|
217
254
|
}
|
|
218
255
|
const reusesSharedKV = sharedKVEntry != null;
|
|
219
256
|
retainSharedKvBuffers = reusesSharedKV || storeSharedKV;
|
|
220
|
-
({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV } = await projectAttentionQKV({
|
|
257
|
+
({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey } = await projectAttentionQKV({
|
|
221
258
|
recorder,
|
|
222
259
|
normed,
|
|
223
260
|
layerWeights,
|
|
@@ -242,7 +279,33 @@ export async function recordLayerAttentionGPU(
|
|
|
242
279
|
trace.attn(layerIdx, `Using fused QKV path: ${qSizeFused}+${kSizeFused}+${vSizeFused}=${totalSize}`);
|
|
243
280
|
}
|
|
244
281
|
: null,
|
|
282
|
+
fusedNormWeight: fusedNormWeightRec,
|
|
283
|
+
fusedNormEps: fusedNormEpsRec,
|
|
284
|
+
fusedNormOffset: fusedNormOffsetRec,
|
|
245
285
|
}));
|
|
286
|
+
// Deferred release of the norm weight buffer for fused path.
|
|
287
|
+
if (fusedNormWeightRec && fusedNormOwnedRec) {
|
|
288
|
+
releaseOrTrack(recorder, fusedNormWeightRec);
|
|
289
|
+
}
|
|
290
|
+
await runProbes('q_proj', qTensor.buffer, {
|
|
291
|
+
layerIdx, numTokens, hiddenSize: numHeads * headDim,
|
|
292
|
+
probes: state.debugProbes, recorder,
|
|
293
|
+
operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
|
|
294
|
+
});
|
|
295
|
+
if (kTensor) {
|
|
296
|
+
await runProbes('k_proj', kTensor.buffer, {
|
|
297
|
+
layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
|
|
298
|
+
probes: state.debugProbes, recorder,
|
|
299
|
+
operatorDiagnostics: state.operatorDiagnostics, dtype: kTensor.dtype,
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
if (vTensor) {
|
|
303
|
+
await runProbes('v_proj', vTensor.buffer, {
|
|
304
|
+
layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
|
|
305
|
+
probes: state.debugProbes, recorder,
|
|
306
|
+
operatorDiagnostics: state.operatorDiagnostics, dtype: vTensor.dtype,
|
|
307
|
+
});
|
|
308
|
+
}
|
|
246
309
|
|
|
247
310
|
// Optional per-head Q/K normalization.
|
|
248
311
|
// Some models use RMSNorm with (1+weight) offset formula, controlled by rmsNormWeightOffset.
|
|
@@ -264,9 +327,23 @@ export async function recordLayerAttentionGPU(
|
|
|
264
327
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
265
328
|
releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
|
|
266
329
|
skipKNorm: reusesSharedKV,
|
|
330
|
+
retainKInput: valueAliasesKey,
|
|
267
331
|
}));
|
|
332
|
+
await runProbes('q_norm', qTensor.buffer, {
|
|
333
|
+
layerIdx, numTokens, hiddenSize: numHeads * headDim,
|
|
334
|
+
probes: state.debugProbes, recorder,
|
|
335
|
+
operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
|
|
336
|
+
});
|
|
337
|
+
if (kTensor) {
|
|
338
|
+
await runProbes('k_norm', kTensor.buffer, {
|
|
339
|
+
layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
|
|
340
|
+
probes: state.debugProbes, recorder,
|
|
341
|
+
operatorDiagnostics: state.operatorDiagnostics, dtype: kTensor.dtype,
|
|
342
|
+
});
|
|
343
|
+
}
|
|
268
344
|
|
|
269
345
|
if (config.valueNorm === true && !reusesSharedKV) {
|
|
346
|
+
const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
|
|
270
347
|
vTensor = await applyAttentionValueNorm({
|
|
271
348
|
recorder,
|
|
272
349
|
vTensor,
|
|
@@ -274,7 +351,16 @@ export async function recordLayerAttentionGPU(
|
|
|
274
351
|
numTokens,
|
|
275
352
|
numKVHeads,
|
|
276
353
|
headDim,
|
|
277
|
-
releaseTemporary: (buffer) =>
|
|
354
|
+
releaseTemporary: (buffer) => {
|
|
355
|
+
if (!valueNormInputAliasesKey) {
|
|
356
|
+
releaseOrTrack(recorder, buffer);
|
|
357
|
+
}
|
|
358
|
+
},
|
|
359
|
+
});
|
|
360
|
+
await runProbes('v_norm', vTensor.buffer, {
|
|
361
|
+
layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
|
|
362
|
+
probes: state.debugProbes, recorder,
|
|
363
|
+
operatorDiagnostics: state.operatorDiagnostics, dtype: vTensor.dtype,
|
|
278
364
|
});
|
|
279
365
|
}
|
|
280
366
|
|
|
@@ -287,6 +373,7 @@ export async function recordLayerAttentionGPU(
|
|
|
287
373
|
numHeads,
|
|
288
374
|
headDim,
|
|
289
375
|
rotaryDim: config.ropeRotaryDim,
|
|
376
|
+
pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
|
|
290
377
|
interleaved: config.ropeInterleaved,
|
|
291
378
|
startPos: currentSeqLen,
|
|
292
379
|
executionPolicies: state.executionPolicies ?? null,
|
|
@@ -296,6 +383,7 @@ export async function recordLayerAttentionGPU(
|
|
|
296
383
|
numHeads: numKVHeads,
|
|
297
384
|
headDim,
|
|
298
385
|
rotaryDim: config.ropeRotaryDim,
|
|
386
|
+
pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
|
|
299
387
|
interleaved: config.ropeInterleaved,
|
|
300
388
|
startPos: currentSeqLen,
|
|
301
389
|
executionPolicies: state.executionPolicies ?? null,
|
|
@@ -507,6 +595,15 @@ export async function recordLayerAttentionGPU(
|
|
|
507
595
|
if (kTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(kCasted.buffer);
|
|
508
596
|
if (vTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(vCasted.buffer);
|
|
509
597
|
}
|
|
598
|
+
// Session precedence is runtime-over-manifest per config-style-guide
|
|
599
|
+
// §Category Rules. getRuntimeConfig() returns the merged session (manifest
|
|
600
|
+
// is the base layer, runtime fields win field-by-field via merge.js).
|
|
601
|
+
// Kernel enforces head_dim=256, f16 KV, contiguous layout; only applies
|
|
602
|
+
// when numTokens > 1 (prefill). Same flag semantics as the non-recorder
|
|
603
|
+
// path in ./run.js.
|
|
604
|
+
const mergedSessionRec = getRuntimeConfig()?.inference?.session;
|
|
605
|
+
const useFlashPrefillRec = mergedSessionRec?.useFlashPrefillAttention === true && numTokens > 1;
|
|
606
|
+
const useOrtFlashPrefillRec = mergedSessionRec?.useOrtFlashPrefillAttention === true && numTokens > 1;
|
|
510
607
|
return recordAttention(recorder, qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
|
|
511
608
|
seqLen: numTokens,
|
|
512
609
|
kvLen: kvState.kvLenForAttention,
|
|
@@ -524,6 +621,8 @@ export async function recordLayerAttentionGPU(
|
|
|
524
621
|
kvPageTable: kvState.kvPageTable,
|
|
525
622
|
kvPageSize: kvState.kvPageSize,
|
|
526
623
|
kernelPath,
|
|
624
|
+
useFlashPrefill: useFlashPrefillRec,
|
|
625
|
+
useOrtFlashPrefill: useOrtFlashPrefillRec,
|
|
527
626
|
});
|
|
528
627
|
},
|
|
529
628
|
};
|
|
@@ -533,13 +632,25 @@ export async function recordLayerAttentionGPU(
|
|
|
533
632
|
}
|
|
534
633
|
|
|
535
634
|
attnOutput = await runAttentionKernel();
|
|
635
|
+
await runProbes('attn_core_out', attnOutput.buffer, {
|
|
636
|
+
layerIdx,
|
|
637
|
+
numTokens,
|
|
638
|
+
hiddenSize: numHeads * headDim,
|
|
639
|
+
probes: state.debugProbes,
|
|
640
|
+
recorder,
|
|
641
|
+
operatorDiagnostics: state.operatorDiagnostics,
|
|
642
|
+
dtype: attnOutput.dtype,
|
|
643
|
+
});
|
|
536
644
|
|
|
537
645
|
attnForProjection = attnOutput;
|
|
538
646
|
if (qGateTensor) {
|
|
647
|
+
// Mirror run.js gate dispatch. Qwen3_5/Qwen 3.6 full attention applies
|
|
648
|
+
// sigmoid(gate) even when the HF config surfaces `output_gate_type=swish`.
|
|
649
|
+
const gateActivation = 'sigmoid';
|
|
539
650
|
attnForProjection = await recordSiLU(recorder, attnOutput, {
|
|
540
651
|
size: numTokens * numHeads * headDim,
|
|
541
652
|
gate: qGateTensor,
|
|
542
|
-
gateActivation
|
|
653
|
+
gateActivation,
|
|
543
654
|
inputActivation: 'identity',
|
|
544
655
|
swigluLimit: null,
|
|
545
656
|
});
|
|
@@ -659,7 +770,9 @@ export async function recordLayerAttentionGPU(
|
|
|
659
770
|
}
|
|
660
771
|
if (!retainSharedKvBuffers) {
|
|
661
772
|
recorder.trackTemporaryBuffer(kTensor.buffer);
|
|
662
|
-
|
|
773
|
+
if (vTensor.buffer !== kTensor.buffer) {
|
|
774
|
+
recorder.trackTemporaryBuffer(vTensor.buffer);
|
|
775
|
+
}
|
|
663
776
|
}
|
|
664
777
|
for (const buffer of buffersToTrack) {
|
|
665
778
|
recorder.trackTemporaryBuffer(buffer);
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { LayerWeights } from '../types.js';
|
|
2
|
+
|
|
3
|
+
export declare function hasQ4KMaterialization(weight: unknown): boolean;
|
|
4
|
+
|
|
5
|
+
export declare function canUseRmsNormWideTileProjectionFusion(
|
|
6
|
+
layerWeights: LayerWeights | null | undefined,
|
|
7
|
+
reusesSharedKV: boolean
|
|
8
|
+
): boolean;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { isWeightBuffer } from '../../../../gpu/weight-buffer.js';
|
|
2
|
+
|
|
3
|
+
export function hasQ4KMaterialization(weight) {
|
|
4
|
+
return isWeightBuffer(weight) && Boolean(weight.materializations?.q4k?.buffer);
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export function canUseRmsNormWideTileProjectionFusion(layerWeights, reusesSharedKV) {
|
|
8
|
+
if (hasQ4KMaterialization(layerWeights?.qkvProj)) {
|
|
9
|
+
return true;
|
|
10
|
+
}
|
|
11
|
+
if (!hasQ4KMaterialization(layerWeights?.qProj)) {
|
|
12
|
+
return false;
|
|
13
|
+
}
|
|
14
|
+
if (reusesSharedKV) {
|
|
15
|
+
return true;
|
|
16
|
+
}
|
|
17
|
+
return hasQ4KMaterialization(layerWeights?.kProj)
|
|
18
|
+
&& (layerWeights?.vProj == null || hasQ4KMaterialization(layerWeights.vProj));
|
|
19
|
+
}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import { isGpuBufferInstance, isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
|
|
4
4
|
import { getDevice } from '../../../../gpu/device.js';
|
|
5
|
+
import { getRuntimeConfig } from '../../../../config/runtime.js';
|
|
5
6
|
import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
|
|
6
7
|
import {
|
|
7
8
|
runMatmul,
|
|
@@ -58,6 +59,7 @@ import {
|
|
|
58
59
|
resolveAttentionPrecisionContract,
|
|
59
60
|
isAttentionKvDtypeExplicit,
|
|
60
61
|
} from './precision-contract.js';
|
|
62
|
+
import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
|
|
61
63
|
|
|
62
64
|
const ATTENTION_DTYPE_LOGGED = new Set();
|
|
63
65
|
|
|
@@ -140,6 +142,7 @@ export async function runLayerAttentionGPU(
|
|
|
140
142
|
let output = null;
|
|
141
143
|
let finalOutput = null;
|
|
142
144
|
let oProjInputTemp = null;
|
|
145
|
+
let valueAliasesKey = false;
|
|
143
146
|
if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
|
|
144
147
|
assertAttentionDtypeTransitionAllowed(
|
|
145
148
|
state,
|
|
@@ -192,9 +195,36 @@ export async function runLayerAttentionGPU(
|
|
|
192
195
|
const reusesSharedKV = sharedKVEntry != null;
|
|
193
196
|
|
|
194
197
|
// 1. Input norm
|
|
195
|
-
|
|
198
|
+
// Opt-in fusion: when useFusedRmsnormWideTile is set AND preconditions match,
|
|
199
|
+
// defer the rmsnorm into each q/k/v_proj matmul via a dedicated WideTile
|
|
200
|
+
// variant. Saves one standalone rmsnorm dispatch per layer.
|
|
201
|
+
let fusedNormWeight = null;
|
|
202
|
+
let fusedNormEps = null;
|
|
203
|
+
let fusedNormOffset = false;
|
|
204
|
+
let fusedNormWeightOwned = false;
|
|
205
|
+
// state.runtimeConfig.inference.session is a stale snapshot ({}); the live
|
|
206
|
+
// module-level runtime config carries the merged profile/override session.
|
|
207
|
+
const rmsNormFusionFlag = getRuntimeConfig()?.inference?.session?.useFusedRmsnormWideTile === true;
|
|
208
|
+
const hasInputNormWeight = layerWeights?.inputNorm != null && getNormWeightBuffer != null;
|
|
209
|
+
const canSelectFusedRmsNormProjection = canUseRmsNormWideTileProjectionFusion(layerWeights, reusesSharedKV);
|
|
210
|
+
const canFuseInputNormIntoProjections = rmsNormFusionFlag
|
|
211
|
+
&& hasInputNormWeight
|
|
212
|
+
&& canSelectFusedRmsNormProjection
|
|
213
|
+
&& !skipInputNorm
|
|
214
|
+
&& isPrefill
|
|
215
|
+
&& numTokens > 1
|
|
216
|
+
&& attentionInput.dtype === 'f32';
|
|
217
|
+
|
|
196
218
|
try {
|
|
197
|
-
if (
|
|
219
|
+
if (canFuseInputNormIntoProjections) {
|
|
220
|
+
const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
221
|
+
fusedNormWeight = normWeightBuf;
|
|
222
|
+
fusedNormEps = rmsNormEps;
|
|
223
|
+
fusedNormOffset = config.rmsNormWeightOffset === true;
|
|
224
|
+
fusedNormWeightOwned = !isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm);
|
|
225
|
+
// Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs the
|
|
226
|
+
// norm internally via the fused kernel variant.
|
|
227
|
+
} else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
|
|
198
228
|
const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
199
229
|
try {
|
|
200
230
|
// Debug: norm weights for configured layers
|
|
@@ -275,7 +305,7 @@ export async function runLayerAttentionGPU(
|
|
|
275
305
|
}),
|
|
276
306
|
});
|
|
277
307
|
let usedFusedQKV = false;
|
|
278
|
-
({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV } = await projectAttentionQKV({
|
|
308
|
+
({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey } = await projectAttentionQKV({
|
|
279
309
|
recorder: null,
|
|
280
310
|
normed,
|
|
281
311
|
layerWeights,
|
|
@@ -300,7 +330,14 @@ export async function runLayerAttentionGPU(
|
|
|
300
330
|
trace.attn(layerIdx, `Using fused QKV path: ${qSizeFused}+${kSizeFused}+${vSizeFused}=${totalSize}`);
|
|
301
331
|
}
|
|
302
332
|
: null,
|
|
333
|
+
fusedNormWeight,
|
|
334
|
+
fusedNormEps,
|
|
335
|
+
fusedNormOffset,
|
|
303
336
|
}));
|
|
337
|
+
// Release the in-flight norm weight buffer used by the fused projections.
|
|
338
|
+
if (fusedNormWeight && fusedNormWeightOwned) {
|
|
339
|
+
releaseBuffer(fusedNormWeight);
|
|
340
|
+
}
|
|
304
341
|
|
|
305
342
|
// Trace Q/K/V projections
|
|
306
343
|
if (kernelTrace.enabled) {
|
|
@@ -413,9 +450,11 @@ export async function runLayerAttentionGPU(
|
|
|
413
450
|
});
|
|
414
451
|
}
|
|
415
452
|
: null,
|
|
453
|
+
retainKInput: valueAliasesKey,
|
|
416
454
|
}));
|
|
417
455
|
|
|
418
456
|
if (config.valueNorm === true && !reusesSharedKV) {
|
|
457
|
+
const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
|
|
419
458
|
vTensor = await applyAttentionValueNorm({
|
|
420
459
|
recorder: null,
|
|
421
460
|
vTensor,
|
|
@@ -423,7 +462,19 @@ export async function runLayerAttentionGPU(
|
|
|
423
462
|
numTokens,
|
|
424
463
|
numKVHeads,
|
|
425
464
|
headDim,
|
|
426
|
-
releaseTemporary: (buffer) =>
|
|
465
|
+
releaseTemporary: (buffer) => {
|
|
466
|
+
if (!valueNormInputAliasesKey) {
|
|
467
|
+
releaseBuffer(buffer);
|
|
468
|
+
}
|
|
469
|
+
},
|
|
470
|
+
});
|
|
471
|
+
await runProbes('v_norm', vTensor.buffer, {
|
|
472
|
+
layerIdx,
|
|
473
|
+
numTokens,
|
|
474
|
+
hiddenSize: numKVHeads * headDim,
|
|
475
|
+
probes: state.debugProbes,
|
|
476
|
+
operatorDiagnostics: state.operatorDiagnostics,
|
|
477
|
+
dtype: vTensor.dtype,
|
|
427
478
|
});
|
|
428
479
|
}
|
|
429
480
|
|
|
@@ -456,6 +507,7 @@ export async function runLayerAttentionGPU(
|
|
|
456
507
|
numHeads,
|
|
457
508
|
headDim,
|
|
458
509
|
rotaryDim: config.ropeRotaryDim,
|
|
510
|
+
pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
|
|
459
511
|
interleaved: config.ropeInterleaved,
|
|
460
512
|
startPos: currentSeqLen,
|
|
461
513
|
executionPolicies: state.executionPolicies ?? null,
|
|
@@ -465,6 +517,7 @@ export async function runLayerAttentionGPU(
|
|
|
465
517
|
numHeads: numKVHeads,
|
|
466
518
|
headDim,
|
|
467
519
|
rotaryDim: config.ropeRotaryDim,
|
|
520
|
+
pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
|
|
468
521
|
interleaved: config.ropeInterleaved,
|
|
469
522
|
startPos: currentSeqLen,
|
|
470
523
|
executionPolicies: state.executionPolicies ?? null,
|
|
@@ -763,6 +816,15 @@ export async function runLayerAttentionGPU(
|
|
|
763
816
|
kForAttn = createTensor(kCasted.buffer, kCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_K');
|
|
764
817
|
vForAttn = createTensor(vCasted.buffer, vCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_V');
|
|
765
818
|
}
|
|
819
|
+
// Session precedence is runtime-over-manifest per config-style-guide
|
|
820
|
+
// §Category Rules: `resolvedSession = merge(manifest.session, runtime.session)`
|
|
821
|
+
// with runtime fields winning. getRuntimeConfig() returns that merged
|
|
822
|
+
// session (state.runtimeConfig.inference.session is a stale snapshot —
|
|
823
|
+
// see comment above at the rmsNorm fusion check). The kernel itself
|
|
824
|
+
// enforces remaining preconditions (head_dim=256, etc.).
|
|
825
|
+
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
826
|
+
const useFlashPrefill = mergedSession?.useFlashPrefillAttention === true && numTokens > 1;
|
|
827
|
+
const useOrtFlashPrefill = mergedSession?.useOrtFlashPrefillAttention === true && numTokens > 1;
|
|
766
828
|
const result = await runAttention(qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
|
|
767
829
|
seqLen: numTokens,
|
|
768
830
|
kvLen: kvState.kvLenForAttention,
|
|
@@ -780,6 +842,8 @@ export async function runLayerAttentionGPU(
|
|
|
780
842
|
kvPageTable: kvState.kvPageTable,
|
|
781
843
|
kvPageSize: kvState.kvPageSize,
|
|
782
844
|
kernelPath,
|
|
845
|
+
useFlashPrefill,
|
|
846
|
+
useOrtFlashPrefill,
|
|
783
847
|
});
|
|
784
848
|
if (prefillFallbackNeedsCast) {
|
|
785
849
|
if (kTensor.dtype !== 'f16') releaseBuffer(kForAttn.buffer);
|
|
@@ -795,6 +859,14 @@ export async function runLayerAttentionGPU(
|
|
|
795
859
|
}
|
|
796
860
|
|
|
797
861
|
attnOutput = await runAttentionKernel();
|
|
862
|
+
await runProbes('attn_core_out', attnOutput.buffer, {
|
|
863
|
+
layerIdx,
|
|
864
|
+
numTokens,
|
|
865
|
+
hiddenSize: numHeads * headDim,
|
|
866
|
+
probes: state.debugProbes,
|
|
867
|
+
operatorDiagnostics: state.operatorDiagnostics,
|
|
868
|
+
dtype: attnOutput.dtype,
|
|
869
|
+
});
|
|
798
870
|
|
|
799
871
|
// Trace attention output
|
|
800
872
|
if (kernelTrace.enabled) {
|
|
@@ -819,10 +891,27 @@ export async function runLayerAttentionGPU(
|
|
|
819
891
|
|
|
820
892
|
attnForProjection = attnOutput;
|
|
821
893
|
if (qGateTensor) {
|
|
894
|
+
// The Qwen3_5 reference (HF transformers) full-attention path
|
|
895
|
+
// applies sigmoid(gate) — NOT silu(gate) — to the attention
|
|
896
|
+
// output regardless of how the architecture's gate label is
|
|
897
|
+
// surfaced in the HF config. See Qwen3_5Attention.forward in
|
|
898
|
+
// transformers/models/qwen3_5/modeling_qwen3_5.py:
|
|
899
|
+
// attn_output = attn_output * torch.sigmoid(gate)
|
|
900
|
+
// The earlier `swish` -> `silu` mapping here was incorrect and
|
|
901
|
+
// produced sign-flipped output for any token whose gate value
|
|
902
|
+
// was negative (silu is sign-preserving via x * sigmoid(x);
|
|
903
|
+
// sigmoid is unconditionally in (0, 1)). At Qwen 3.6 27B scale
|
|
904
|
+
// this drove residual-stream divergence by L=3 (HF reference vs
|
|
905
|
+
// upstream rel_l2≈1.37, mean per-token correlation ≈ -0.96 — a
|
|
906
|
+
// sign-flip pattern). config.outputGateType is retained for
|
|
907
|
+
// future architectures whose gate truly is silu, but Qwen 3.6's
|
|
908
|
+
// `output_gate_type=swish` still routes here as sigmoid because
|
|
909
|
+
// that is what the reference forward computes.
|
|
910
|
+
const gateActivation = 'sigmoid';
|
|
822
911
|
attnForProjection = await runSiLU(attnOutput, {
|
|
823
912
|
size: numTokens * numHeads * headDim,
|
|
824
913
|
gate: qGateTensor,
|
|
825
|
-
gateActivation
|
|
914
|
+
gateActivation,
|
|
826
915
|
inputActivation: 'identity',
|
|
827
916
|
swigluLimit: null,
|
|
828
917
|
});
|
|
@@ -973,7 +1062,9 @@ export async function runLayerAttentionGPU(
|
|
|
973
1062
|
}
|
|
974
1063
|
if (!reusesSharedKV && !storeSharedKV) {
|
|
975
1064
|
releaseBuffer(kTensor.buffer);
|
|
976
|
-
|
|
1065
|
+
if (vTensor.buffer !== kTensor.buffer) {
|
|
1066
|
+
releaseBuffer(vTensor.buffer);
|
|
1067
|
+
}
|
|
977
1068
|
}
|
|
978
1069
|
for (const buffer of buffersToRelease) {
|
|
979
1070
|
releaseBuffer(buffer);
|
|
@@ -56,6 +56,8 @@ export interface AttentionConfig {
|
|
|
56
56
|
} | null;
|
|
57
57
|
/** RoPE rotary dimension (may differ from headDim with partial rotary). */
|
|
58
58
|
ropeRotaryDim?: number;
|
|
59
|
+
/** RoPE frequency base dimension; non-interleaved kernels use this as the rotate-half pair span. */
|
|
60
|
+
ropeFrequencyBaseDim?: number;
|
|
59
61
|
/** Whether RoPE uses interleaved layout. */
|
|
60
62
|
ropeInterleaved?: boolean;
|
|
61
63
|
/** Token IDs for the current micro-batch (required by BDPA KV ingestion). */
|
|
@@ -44,16 +44,21 @@ export function releaseOrTrack(recorder, buffer) {
|
|
|
44
44
|
// ============================================================================
|
|
45
45
|
|
|
46
46
|
|
|
47
|
-
const qkNormOnesCache = new
|
|
47
|
+
const qkNormOnesCache = new WeakMap();
|
|
48
48
|
|
|
49
49
|
|
|
50
50
|
export function getQKNormOnesBuffer(headDim) {
|
|
51
|
-
const cached = qkNormOnesCache.get(headDim);
|
|
52
|
-
if (cached) return cached;
|
|
53
51
|
const device = getDevice();
|
|
54
52
|
if (!device) {
|
|
55
53
|
throw new Error('No GPU device available for Q/K norm buffer');
|
|
56
54
|
}
|
|
55
|
+
let perDeviceCache = qkNormOnesCache.get(device);
|
|
56
|
+
if (!perDeviceCache) {
|
|
57
|
+
perDeviceCache = new Map();
|
|
58
|
+
qkNormOnesCache.set(device, perDeviceCache);
|
|
59
|
+
}
|
|
60
|
+
const cached = perDeviceCache.get(headDim);
|
|
61
|
+
if (cached) return cached;
|
|
57
62
|
const data = new Float32Array(headDim);
|
|
58
63
|
data.fill(1);
|
|
59
64
|
const buffer = device.createBuffer({
|
|
@@ -62,6 +67,6 @@ export function getQKNormOnesBuffer(headDim) {
|
|
|
62
67
|
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
|
|
63
68
|
});
|
|
64
69
|
device.queue.writeBuffer(buffer, 0, data);
|
|
65
|
-
|
|
70
|
+
perDeviceCache.set(headDim, buffer);
|
|
66
71
|
return buffer;
|
|
67
72
|
}
|
|
@@ -36,18 +36,8 @@ export type ChatTemplateType =
|
|
|
36
36
|
| 'translategemma'
|
|
37
37
|
| null;
|
|
38
38
|
|
|
39
|
-
export declare function formatGemmaChat(messages: ChatMessage[]): string;
|
|
40
|
-
|
|
41
39
|
export interface ChatFormatOptions {
|
|
42
40
|
thinking?: boolean;
|
|
43
41
|
}
|
|
44
42
|
|
|
45
|
-
export declare function formatGemma4Chat(messages: ChatMessage[], options?: ChatFormatOptions): string;
|
|
46
|
-
|
|
47
|
-
export declare function formatLlama3Chat(messages: ChatMessage[]): string;
|
|
48
|
-
|
|
49
|
-
export declare function formatGptOssChat(messages: ChatMessage[]): string;
|
|
50
|
-
|
|
51
|
-
export declare function formatTranslateGemmaChat(messages: ChatMessage[]): string;
|
|
52
|
-
|
|
53
43
|
export declare function formatChatMessages(messages: ChatMessage[], templateType?: ChatTemplateType, options?: ChatFormatOptions): string;
|
|
@@ -229,7 +229,7 @@ function formatGemma4(messages, options) {
|
|
|
229
229
|
if (options?.thinking === true) {
|
|
230
230
|
parts.push('<|turn>model\n<|think|>\n');
|
|
231
231
|
} else {
|
|
232
|
-
parts.push('<|turn>model\n');
|
|
232
|
+
parts.push('<|turn>model\n<|channel>thought\n<channel|>');
|
|
233
233
|
}
|
|
234
234
|
return parts.join('');
|
|
235
235
|
}
|
|
@@ -440,10 +440,3 @@ export function formatChatMessages(messages, templateType, options) {
|
|
|
440
440
|
}
|
|
441
441
|
throw new Error(`Unrecognized chat template type: ${templateType}`);
|
|
442
442
|
}
|
|
443
|
-
|
|
444
|
-
export const formatGemmaChat = formatTurnBased;
|
|
445
|
-
export const formatGemma4Chat = formatGemma4;
|
|
446
|
-
export const formatLlama3Chat = formatHeaderBased;
|
|
447
|
-
export const formatGptOssChat = formatChannelBased;
|
|
448
|
-
export const formatQwenChat = formatQwen;
|
|
449
|
-
export const formatTranslateGemmaChat = formatTranslateGemma;
|
|
@@ -46,6 +46,8 @@ export interface RawConfig {
|
|
|
46
46
|
n_head?: number;
|
|
47
47
|
attentionHeadCount?: number;
|
|
48
48
|
num_key_value_heads?: number;
|
|
49
|
+
num_global_key_value_heads?: number;
|
|
50
|
+
num_global_kv_heads?: number;
|
|
49
51
|
attentionHeadCountKV?: number;
|
|
50
52
|
head_dim?: number;
|
|
51
53
|
global_head_dim?: number;
|
|
@@ -109,12 +111,6 @@ export interface TensorInfo {
|
|
|
109
111
|
|
|
110
112
|
export type Manifest = RuntimeModelContract;
|
|
111
113
|
|
|
112
|
-
export interface AttentionParams {
|
|
113
|
-
numHeads: number;
|
|
114
|
-
numKVHeads: number;
|
|
115
|
-
headDim: number;
|
|
116
|
-
}
|
|
117
|
-
|
|
118
114
|
export interface ParsedModelConfig {
|
|
119
115
|
numLayers: number;
|
|
120
116
|
hiddenSize: number;
|
|
@@ -123,6 +119,7 @@ export interface ParsedModelConfig {
|
|
|
123
119
|
maxIntermediateSize: number;
|
|
124
120
|
numHeads: number;
|
|
125
121
|
numKVHeads: number;
|
|
122
|
+
numGlobalKVHeads: number | null;
|
|
126
123
|
headDim: number;
|
|
127
124
|
globalHeadDim: number | null;
|
|
128
125
|
vocabSize: number;
|
|
@@ -250,12 +247,6 @@ export interface AudioEncoderConfig {
|
|
|
250
247
|
export function getStopTokenIds(manifest: Manifest): number[];
|
|
251
248
|
export function resolveLayerIntermediateSize(config: ParsedModelConfig, layerIdx: number): number;
|
|
252
249
|
|
|
253
|
-
/**
|
|
254
|
-
* Validate that a chatTemplate.type is a known formatter type.
|
|
255
|
-
* Logs a warning for unknown types. Returns true if valid or null.
|
|
256
|
-
*/
|
|
257
|
-
export function validateChatTemplateType(type: string | null | undefined, modelId: string): boolean;
|
|
258
|
-
|
|
259
250
|
/**
|
|
260
251
|
* Extended manifest with inference config for manifest-first parsing.
|
|
261
252
|
*/
|
|
@@ -270,24 +261,11 @@ export interface ManifestWithInference {
|
|
|
270
261
|
eos_token_id: number | number[];
|
|
271
262
|
}
|
|
272
263
|
|
|
273
|
-
/**
|
|
274
|
-
* Check if manifest has inference config for manifest-first parsing.
|
|
275
|
-
*/
|
|
276
|
-
export function hasManifestInference(manifest: Manifest): manifest is Manifest & { inference: ManifestInferenceSchema };
|
|
277
|
-
|
|
278
264
|
export function validateRequiredInferenceFields(
|
|
279
265
|
inf: ManifestInferenceSchema,
|
|
280
266
|
modelId: string
|
|
281
267
|
): void;
|
|
282
268
|
|
|
283
|
-
/**
|
|
284
|
-
* Convert MergedConfig to ParsedModelConfig.
|
|
285
|
-
*/
|
|
286
|
-
export function toParsedConfigFromMerged(
|
|
287
|
-
merged: MergedConfig,
|
|
288
|
-
manifest: ManifestWithInference
|
|
289
|
-
): ParsedModelConfig;
|
|
290
|
-
|
|
291
269
|
/**
|
|
292
270
|
* Parse model config from manifest using manifest-first resolution.
|
|
293
271
|
*/
|