doppler-gpu 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +210 -0
- package/README.md +63 -38
- package/assets/doppler.svg +57 -1601
- package/package.json +111 -32
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +486 -0
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +885 -744
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +469 -43
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +66 -10
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +69 -4
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +34 -50
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +35 -2
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +148 -3
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +8 -60
- package/src/client/wrap-pipeline-handle.d.ts +82 -0
- package/src/client/wrap-pipeline-handle.js +259 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +28 -157
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +163 -126
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +3593 -1146
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +3 -1
- package/src/config/merge-helpers.js +2 -1
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +65 -7
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +20 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +16 -0
- package/src/config/schema/converter.schema.js +11 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +13 -2
- package/src/config/schema/doppler.schema.js +140 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +39 -1
- package/src/config/schema/execution-v1.schema.js +35 -20
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +14 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +175 -2
- package/src/config/schema/manifest.schema.js +58 -3
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -9
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +183 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
- package/src/config/transforms/execution-graph-transforms.js +829 -168
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +31 -14
- package/src/converter/core.d.ts +65 -0
- package/src/converter/core.js +1153 -22
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +218 -5
- package/src/converter/quantizer.d.ts +24 -4
- package/src/converter/quantizer.js +92 -2
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +296 -38
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1733 -58
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +5 -14
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +192 -12
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +111 -2
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +10 -0
- package/src/formats/rdrr/index.js +10 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +26 -6
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +193 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +111 -2
- package/src/formats/rdrr/validation.js +260 -1
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +45 -23
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +130 -14
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +1 -8
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +458 -16
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +13 -120
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +84 -8
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +25 -22
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +77 -5
- package/src/gpu/kernels/index.js +55 -4
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +44 -2
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +172 -10
- package/src/gpu/kernels/matmul.d.ts +1 -29
- package/src/gpu/kernels/matmul.js +240 -38
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +1 -56
- package/src/gpu/kernels/moe.js +46 -33
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -12
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +6 -4
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +30 -7
- package/src/gpu/kernels/rmsnorm.js +49 -8
- package/src/gpu/kernels/rmsnorm.wgsl +15 -12
- package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +28 -21
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +8 -223
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +43 -0
- package/src/gpu/kernels/utils.js +81 -11
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +10 -12
- package/src/gpu/uniform-cache.js +25 -30
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
- package/src/inference/browser-harness-suite-helpers.js +525 -3
- package/src/inference/browser-harness-text-helpers.d.ts +89 -0
- package/src/inference/browser-harness-text-helpers.js +459 -25
- package/src/inference/browser-harness.d.ts +7 -28
- package/src/inference/browser-harness.js +556 -79
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +31 -0
- package/src/inference/kv-cache/base.js +125 -2
- package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
- package/src/inference/kv-cache/mixed-geometry.js +121 -4
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
- package/src/inference/pipelines/text/attention/projections.js +398 -71
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +542 -56
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +343 -50
- package/src/inference/pipelines/text/attention/types.d.ts +7 -1
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +25 -26
- package/src/inference/pipelines/text/config.js +304 -25
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +25 -29
- package/src/inference/pipelines/text/embed.js +16 -45
- package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
- package/src/inference/pipelines/text/execution-plan.js +21 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
- package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
- package/src/inference/pipelines/text/execution-v1.js +461 -158
- package/src/inference/pipelines/text/ffn/dense.js +214 -6
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +84 -17
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +9 -7
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
- package/src/inference/pipelines/text/generator-steps.js +275 -79
- package/src/inference/pipelines/text/generator.d.ts +38 -5
- package/src/inference/pipelines/text/generator.js +1259 -496
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +13 -3
- package/src/inference/pipelines/text/init.js +213 -725
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +373 -558
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +419 -66
- package/src/inference/pipelines/text/logits/index.d.ts +3 -3
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +42 -4
- package/src/inference/pipelines/text/ops.js +20 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +119 -31
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +153 -9
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
- package/src/inference/pipelines/text/types.d.ts +92 -81
- package/src/inference/pipelines/text/weights.d.ts +4 -52
- package/src/inference/pipelines/text/weights.js +6 -56
- package/src/inference/pipelines/text.d.ts +33 -1
- package/src/inference/pipelines/text.js +375 -55
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +341 -174
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +14 -0
- package/src/loader/doppler-loader.js +378 -25
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +276 -11
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +61 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +52 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +36 -8
- package/src/loader/tensors/tensor-loader.js +521 -22
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +13 -42
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +57 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +162 -10
- package/src/rules/inference/capability-transforms.rules.json +111 -0
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +117 -16
- package/src/rules/inference/ffn.rules.json +24 -2
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +17 -0
- package/src/rules/rule-registry.js +59 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +241 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +31 -4
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +53 -9
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +8 -3
- package/src/tooling/command-api.d.ts +22 -1
- package/src/tooling/command-api.js +11 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +14 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +3 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +571 -37
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +107 -17
- package/src/tooling/node-command-runner.js +43 -2
- package/src/tooling/node-convert-worker-pool.js +15 -0
- package/src/tooling/node-convert-worker.js +11 -1
- package/src/tooling/node-converter.js +102 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +112 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +10 -5
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +110 -8
- package/src/tooling/source-runtime-bundle.d.ts +3 -0
- package/src/tooling/source-runtime-bundle.js +30 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +25 -12
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { LayerWeights } from '../types.js';
|
|
2
|
+
|
|
3
|
+
export declare function hasQ4KMaterialization(weight: unknown): boolean;
|
|
4
|
+
|
|
5
|
+
export declare function canUseRmsNormWideTileProjectionFusion(
|
|
6
|
+
layerWeights: LayerWeights | null | undefined,
|
|
7
|
+
reusesSharedKV: boolean
|
|
8
|
+
): boolean;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { isWeightBuffer } from '../../../../gpu/weight-buffer.js';
|
|
2
|
+
|
|
3
|
+
export function hasQ4KMaterialization(weight) {
|
|
4
|
+
return isWeightBuffer(weight) && Boolean(weight.materializations?.q4k?.buffer);
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export function canUseRmsNormWideTileProjectionFusion(layerWeights, reusesSharedKV) {
|
|
8
|
+
if (hasQ4KMaterialization(layerWeights?.qkvProj)) {
|
|
9
|
+
return true;
|
|
10
|
+
}
|
|
11
|
+
if (!hasQ4KMaterialization(layerWeights?.qProj)) {
|
|
12
|
+
return false;
|
|
13
|
+
}
|
|
14
|
+
if (reusesSharedKV) {
|
|
15
|
+
return true;
|
|
16
|
+
}
|
|
17
|
+
return hasQ4KMaterialization(layerWeights?.kProj)
|
|
18
|
+
&& (layerWeights?.vProj == null || hasQ4KMaterialization(layerWeights.vProj));
|
|
19
|
+
}
|
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
3
|
import { isGpuBufferInstance, isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
|
|
4
|
-
import { getDevice } from '../../../../gpu/device.js';
|
|
4
|
+
import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
|
|
5
|
+
import { getRuntimeConfig } from '../../../../config/runtime.js';
|
|
5
6
|
import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
|
|
6
7
|
import {
|
|
7
8
|
runMatmul,
|
|
8
9
|
runRMSNorm,
|
|
9
10
|
runRoPE,
|
|
11
|
+
canUseRoPEQK,
|
|
12
|
+
runRoPEQK,
|
|
10
13
|
runAttention,
|
|
11
14
|
runAttentionBDPA,
|
|
12
15
|
runAttentionTieredQuant,
|
|
@@ -33,6 +36,9 @@ import {
|
|
|
33
36
|
projectAttentionQKV,
|
|
34
37
|
applyAttentionQKNorm,
|
|
35
38
|
applyAttentionValueNorm,
|
|
39
|
+
hasAttentionProjectionDiagnostics,
|
|
40
|
+
hasAttentionStageDiagnostics,
|
|
41
|
+
resolveAttentionQKNormState,
|
|
36
42
|
} from './projections.js';
|
|
37
43
|
import { prepareAttentionProjectionInput } from './output-projection.js';
|
|
38
44
|
|
|
@@ -46,6 +52,7 @@ import {
|
|
|
46
52
|
} from '../../../../config/kernel-path-loader.js';
|
|
47
53
|
import {
|
|
48
54
|
resolveKVCacheState,
|
|
55
|
+
createDiffusionGemmaDecoderKVState,
|
|
49
56
|
buildAttentionDispatchParams,
|
|
50
57
|
buildAttentionInputsData,
|
|
51
58
|
} from './dispatch-params.js';
|
|
@@ -58,9 +65,30 @@ import {
|
|
|
58
65
|
resolveAttentionPrecisionContract,
|
|
59
66
|
isAttentionKvDtypeExplicit,
|
|
60
67
|
} from './precision-contract.js';
|
|
68
|
+
import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
|
|
61
69
|
|
|
62
70
|
const ATTENTION_DTYPE_LOGGED = new Set();
|
|
63
71
|
|
|
72
|
+
function isWideTileQ4KPhaseEnabled(session, phase) {
|
|
73
|
+
return phase === 'decode'
|
|
74
|
+
? session?.useWideTileQ4KDecode === true
|
|
75
|
+
: session?.useWideTileQ4KPrefill === true;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function canUseQ4KWideTileResidualFusion(options = {}) {
|
|
79
|
+
return options.hasResidual === true
|
|
80
|
+
&& options.oProjDtype === 'q4k'
|
|
81
|
+
&& options.inputDtype === 'f32'
|
|
82
|
+
&& options.outputDtype === 'f32'
|
|
83
|
+
&& options.residualMatches === true
|
|
84
|
+
&& options.hasLoRA !== true
|
|
85
|
+
&& getKernelCapabilities().hasF16 === true
|
|
86
|
+
&& getKernelCapabilities().hasSubgroups === true
|
|
87
|
+
&& options.session?.useWideTileResidualFusion === true
|
|
88
|
+
&& options.session?.retainQ4KMaterialization === true
|
|
89
|
+
&& isWideTileQ4KPhaseEnabled(options.session, options.phase);
|
|
90
|
+
}
|
|
91
|
+
|
|
64
92
|
function assertAttentionDtypeTransitionAllowed(state, fromDtype, toDtype, detail, transitionDeclaredBy = null) {
|
|
65
93
|
assertImplicitDtypeTransitionAllowed({
|
|
66
94
|
executionPolicies: state?.executionPolicies ?? null,
|
|
@@ -106,6 +134,8 @@ export async function runLayerAttentionGPU(
|
|
|
106
134
|
multimodalBidirectionalSpan = null,
|
|
107
135
|
sharedKVSourceLayerIdx = null,
|
|
108
136
|
storeSharedKV = false,
|
|
137
|
+
diffusionGemmaDecoder = false,
|
|
138
|
+
precomputedInputNorm = null,
|
|
109
139
|
} = config;
|
|
110
140
|
|
|
111
141
|
const device = getDevice();
|
|
@@ -128,6 +158,9 @@ export async function runLayerAttentionGPU(
|
|
|
128
158
|
const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: useF16Activations });
|
|
129
159
|
const kvCacheDtype = attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? kvCacheFallback;
|
|
130
160
|
const allowF16Attention = useF16Activations && kvCacheDtype === 'f16';
|
|
161
|
+
const attentionInputTransitionDeclaredBy = attentionPrecisionContract.explicitInputDtype
|
|
162
|
+
? 'step_precision'
|
|
163
|
+
: null;
|
|
131
164
|
let attentionInput = input;
|
|
132
165
|
let attentionInputTemp = false;
|
|
133
166
|
let normed = attentionInput;
|
|
@@ -140,12 +173,17 @@ export async function runLayerAttentionGPU(
|
|
|
140
173
|
let output = null;
|
|
141
174
|
let finalOutput = null;
|
|
142
175
|
let oProjInputTemp = null;
|
|
176
|
+
let valueAliasesKey = false;
|
|
177
|
+
let decoderKVState = null;
|
|
178
|
+
let decoderKTemp = null;
|
|
179
|
+
let decoderVTemp = null;
|
|
143
180
|
if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
|
|
144
181
|
assertAttentionDtypeTransitionAllowed(
|
|
145
182
|
state,
|
|
146
183
|
input.dtype,
|
|
147
184
|
attentionActivationDtype,
|
|
148
|
-
'The attention kernel selection would widen the input implicitly.'
|
|
185
|
+
'The attention kernel selection would widen the input implicitly.',
|
|
186
|
+
attentionInputTransitionDeclaredBy
|
|
149
187
|
);
|
|
150
188
|
attentionInput = attentionActivationDtype === 'f16'
|
|
151
189
|
? await castF32ToF16(input)
|
|
@@ -192,9 +230,57 @@ export async function runLayerAttentionGPU(
|
|
|
192
230
|
const reusesSharedKV = sharedKVEntry != null;
|
|
193
231
|
|
|
194
232
|
// 1. Input norm
|
|
195
|
-
|
|
233
|
+
// Opt-in fusion: when useFusedRmsnormWideTile is set AND preconditions match,
|
|
234
|
+
// defer the rmsnorm into each q/k/v_proj matmul via a dedicated WideTile
|
|
235
|
+
// variant. Saves one standalone rmsnorm dispatch per layer.
|
|
236
|
+
let fusedNormWeight = null;
|
|
237
|
+
let fusedNormEps = null;
|
|
238
|
+
let fusedNormOffset = false;
|
|
239
|
+
let fusedNormWeightOwned = false;
|
|
240
|
+
// state.runtimeConfig.inference.session is a stale snapshot ({}); the live
|
|
241
|
+
// module-level runtime config carries the merged profile/override session.
|
|
242
|
+
const rmsNormFusionFlag = getRuntimeConfig()?.inference?.session?.useFusedRmsnormWideTile === true;
|
|
243
|
+
const hasInputNormWeight = layerWeights?.inputNorm != null && getNormWeightBuffer != null;
|
|
244
|
+
const canSelectFusedRmsNormProjection = canUseRmsNormWideTileProjectionFusion(layerWeights, reusesSharedKV);
|
|
245
|
+
const canFuseInputNormIntoProjections = rmsNormFusionFlag
|
|
246
|
+
&& hasInputNormWeight
|
|
247
|
+
&& canSelectFusedRmsNormProjection
|
|
248
|
+
&& !skipInputNorm
|
|
249
|
+
&& isPrefill
|
|
250
|
+
&& numTokens > 1
|
|
251
|
+
&& attentionInput.dtype === 'f32';
|
|
252
|
+
|
|
196
253
|
try {
|
|
197
|
-
if (
|
|
254
|
+
if (precomputedInputNorm) {
|
|
255
|
+
if (
|
|
256
|
+
isPrefill
|
|
257
|
+
|| numTokens !== 1
|
|
258
|
+
|| skipInputNorm
|
|
259
|
+
|| !layerWeights.inputNorm
|
|
260
|
+
|| !getNormWeightBuffer
|
|
261
|
+
|| precomputedInputNorm.dtype !== attentionInput.dtype
|
|
262
|
+
|| precomputedInputNorm.dtype !== 'f32'
|
|
263
|
+
) {
|
|
264
|
+
throw new Error(`Layer ${layerIdx} received an incompatible precomputed input norm tensor.`);
|
|
265
|
+
}
|
|
266
|
+
normed = precomputedInputNorm;
|
|
267
|
+
await runProbes('post_input_norm', normed.buffer, {
|
|
268
|
+
layerIdx,
|
|
269
|
+
numTokens,
|
|
270
|
+
hiddenSize,
|
|
271
|
+
probes: state.debugProbes,
|
|
272
|
+
operatorDiagnostics: state.operatorDiagnostics,
|
|
273
|
+
dtype: normed.dtype,
|
|
274
|
+
});
|
|
275
|
+
} else if (canFuseInputNormIntoProjections) {
|
|
276
|
+
const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
277
|
+
fusedNormWeight = normWeightBuf;
|
|
278
|
+
fusedNormEps = rmsNormEps;
|
|
279
|
+
fusedNormOffset = config.rmsNormWeightOffset === true;
|
|
280
|
+
fusedNormWeightOwned = !isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm);
|
|
281
|
+
// Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs the
|
|
282
|
+
// norm internally via the fused kernel variant.
|
|
283
|
+
} else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
|
|
198
284
|
const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
199
285
|
try {
|
|
200
286
|
// Debug: norm weights for configured layers
|
|
@@ -275,7 +361,24 @@ export async function runLayerAttentionGPU(
|
|
|
275
361
|
}),
|
|
276
362
|
});
|
|
277
363
|
let usedFusedQKV = false;
|
|
278
|
-
|
|
364
|
+
let qkNormApplied = false;
|
|
365
|
+
let ropeApplied = false;
|
|
366
|
+
const qkNormState = resolveAttentionQKNormState({ config, layerWeights, layerIdx, reusesSharedKV });
|
|
367
|
+
const qkNormProjectionDiagnostics = hasAttentionProjectionDiagnostics(state)
|
|
368
|
+
|| kernelTrace.enabled
|
|
369
|
+
|| isKernelDebugEnabled(layerIdx)
|
|
370
|
+
|| (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !!debugCheckBuffer);
|
|
371
|
+
const qkNormRoPEDiagnostics = hasAttentionStageDiagnostics(
|
|
372
|
+
state,
|
|
373
|
+
['q_proj', 'k_proj', 'v_proj', 'q_norm', 'k_norm']
|
|
374
|
+
)
|
|
375
|
+
|| kernelTrace.enabled
|
|
376
|
+
|| isKernelDebugEnabled(layerIdx)
|
|
377
|
+
|| (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !!debugCheckBuffer);
|
|
378
|
+
const runtimeSession = getRuntimeConfig()?.inference?.session;
|
|
379
|
+
const qkNormFusionFlag = runtimeSession?.useFusedQKVSplitQKNorm === true;
|
|
380
|
+
const qkNormRoPEFusionFlag = runtimeSession?.useFusedQKVSplitQKNormRoPE === true;
|
|
381
|
+
({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey, qkNormApplied, ropeApplied } = await projectAttentionQKV({
|
|
279
382
|
recorder: null,
|
|
280
383
|
normed,
|
|
281
384
|
layerWeights,
|
|
@@ -300,7 +403,43 @@ export async function runLayerAttentionGPU(
|
|
|
300
403
|
trace.attn(layerIdx, `Using fused QKV path: ${qSizeFused}+${kSizeFused}+${vSizeFused}=${totalSize}`);
|
|
301
404
|
}
|
|
302
405
|
: null,
|
|
406
|
+
fusedNormWeight,
|
|
407
|
+
fusedNormEps,
|
|
408
|
+
fusedNormOffset,
|
|
409
|
+
qkNormFusion: {
|
|
410
|
+
enabled: qkNormFusionFlag && qkNormState.wantsQKNorm,
|
|
411
|
+
getNormWeightBuffer,
|
|
412
|
+
rmsNormEps,
|
|
413
|
+
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
414
|
+
skipKNorm: qkNormState.skipKNorm,
|
|
415
|
+
allowUnitQKNorm: qkNormState.allowUnitQKNorm,
|
|
416
|
+
projectionDiagnosticsEnabled: qkNormProjectionDiagnostics,
|
|
417
|
+
},
|
|
418
|
+
qkNormRoPEFusion: {
|
|
419
|
+
enabled: qkNormRoPEFusionFlag
|
|
420
|
+
&& qkNormState.wantsQKNorm
|
|
421
|
+
&& !disableRoPE
|
|
422
|
+
&& !!state.ropeFreqsCos
|
|
423
|
+
&& !!state.ropeFreqsSin,
|
|
424
|
+
getNormWeightBuffer,
|
|
425
|
+
rmsNormEps,
|
|
426
|
+
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
427
|
+
skipKNorm: qkNormState.skipKNorm,
|
|
428
|
+
allowUnitQKNorm: qkNormState.allowUnitQKNorm,
|
|
429
|
+
projectionDiagnosticsEnabled: qkNormRoPEDiagnostics,
|
|
430
|
+
freqsCos: state.ropeFreqsCos,
|
|
431
|
+
freqsSin: state.ropeFreqsSin,
|
|
432
|
+
startPos: currentSeqLen,
|
|
433
|
+
rotaryDim: config.ropeRotaryDim,
|
|
434
|
+
pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
|
|
435
|
+
interleaved: config.ropeInterleaved,
|
|
436
|
+
reusesSharedKV,
|
|
437
|
+
},
|
|
303
438
|
}));
|
|
439
|
+
// Release the in-flight norm weight buffer used by the fused projections.
|
|
440
|
+
if (fusedNormWeight && fusedNormWeightOwned) {
|
|
441
|
+
releaseBuffer(fusedNormWeight);
|
|
442
|
+
}
|
|
304
443
|
|
|
305
444
|
// Trace Q/K/V projections
|
|
306
445
|
if (kernelTrace.enabled) {
|
|
@@ -369,17 +508,19 @@ export async function runLayerAttentionGPU(
|
|
|
369
508
|
}
|
|
370
509
|
|
|
371
510
|
// Optional per-head Q/K normalization
|
|
372
|
-
const wantsQKNorm = config.queryKeyNorm === true;
|
|
373
|
-
const hasQNorm = !!layerWeights.qNorm;
|
|
374
|
-
const hasKNorm = !!layerWeights.kNorm;
|
|
375
511
|
if (isKernelDebugEnabled(layerIdx)) {
|
|
376
|
-
logKernelStep(
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
512
|
+
logKernelStep(
|
|
513
|
+
'qk_norm',
|
|
514
|
+
{
|
|
515
|
+
layerIdx,
|
|
516
|
+
label: `hasQ=${qkNormState.hasQNorm} hasK=${qkNormState.hasKNorm} ` +
|
|
517
|
+
`wants=${qkNormState.wantsQKNorm} unit=${qkNormState.allowUnitQKNorm}`,
|
|
518
|
+
}
|
|
519
|
+
);
|
|
380
520
|
}
|
|
381
521
|
|
|
382
|
-
(
|
|
522
|
+
if (qkNormState.wantsQKNorm && !qkNormApplied) {
|
|
523
|
+
({ qTensor, kTensor } = await applyAttentionQKNorm({
|
|
383
524
|
recorder: null,
|
|
384
525
|
qTensor,
|
|
385
526
|
kTensor,
|
|
@@ -392,7 +533,7 @@ export async function runLayerAttentionGPU(
|
|
|
392
533
|
headDim,
|
|
393
534
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
394
535
|
releaseTemporary: (buffer) => releaseBuffer(buffer),
|
|
395
|
-
skipKNorm:
|
|
536
|
+
skipKNorm: qkNormState.skipKNorm,
|
|
396
537
|
onQNormApplied: isKernelDebugEnabled(layerIdx)
|
|
397
538
|
? async (tensor) => {
|
|
398
539
|
await dumpTokenVector(tensor.buffer, 'Q_norm', {
|
|
@@ -412,10 +553,14 @@ export async function runLayerAttentionGPU(
|
|
|
412
553
|
dtype: tensor.dtype,
|
|
413
554
|
});
|
|
414
555
|
}
|
|
415
|
-
|
|
416
|
-
|
|
556
|
+
: null,
|
|
557
|
+
retainKInput: valueAliasesKey,
|
|
558
|
+
allowUnitQKNorm: qkNormState.allowUnitQKNorm,
|
|
559
|
+
}));
|
|
560
|
+
}
|
|
417
561
|
|
|
418
562
|
if (config.valueNorm === true && !reusesSharedKV) {
|
|
563
|
+
const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
|
|
419
564
|
vTensor = await applyAttentionValueNorm({
|
|
420
565
|
recorder: null,
|
|
421
566
|
vTensor,
|
|
@@ -423,27 +568,41 @@ export async function runLayerAttentionGPU(
|
|
|
423
568
|
numTokens,
|
|
424
569
|
numKVHeads,
|
|
425
570
|
headDim,
|
|
426
|
-
releaseTemporary: (buffer) =>
|
|
571
|
+
releaseTemporary: (buffer) => {
|
|
572
|
+
if (!valueNormInputAliasesKey) {
|
|
573
|
+
releaseBuffer(buffer);
|
|
574
|
+
}
|
|
575
|
+
},
|
|
576
|
+
});
|
|
577
|
+
await runProbes('v_norm', vTensor.buffer, {
|
|
578
|
+
layerIdx,
|
|
579
|
+
numTokens,
|
|
580
|
+
hiddenSize: numKVHeads * headDim,
|
|
581
|
+
probes: state.debugProbes,
|
|
582
|
+
operatorDiagnostics: state.operatorDiagnostics,
|
|
583
|
+
dtype: vTensor.dtype,
|
|
427
584
|
});
|
|
428
585
|
}
|
|
429
586
|
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
numTokens,
|
|
433
|
-
hiddenSize: numHeads * headDim,
|
|
434
|
-
probes: state.debugProbes,
|
|
435
|
-
operatorDiagnostics: state.operatorDiagnostics,
|
|
436
|
-
dtype: qTensor.dtype,
|
|
437
|
-
});
|
|
438
|
-
if (!reusesSharedKV) {
|
|
439
|
-
await runProbes('k_norm', kTensor.buffer, {
|
|
587
|
+
if (qkNormState.wantsQKNorm) {
|
|
588
|
+
await runProbes('q_norm', qTensor.buffer, {
|
|
440
589
|
layerIdx,
|
|
441
590
|
numTokens,
|
|
442
|
-
hiddenSize:
|
|
591
|
+
hiddenSize: numHeads * headDim,
|
|
443
592
|
probes: state.debugProbes,
|
|
444
593
|
operatorDiagnostics: state.operatorDiagnostics,
|
|
445
|
-
dtype:
|
|
594
|
+
dtype: qTensor.dtype,
|
|
446
595
|
});
|
|
596
|
+
if (!reusesSharedKV) {
|
|
597
|
+
await runProbes('k_norm', kTensor.buffer, {
|
|
598
|
+
layerIdx,
|
|
599
|
+
numTokens,
|
|
600
|
+
hiddenSize: numKVHeads * headDim,
|
|
601
|
+
probes: state.debugProbes,
|
|
602
|
+
operatorDiagnostics: state.operatorDiagnostics,
|
|
603
|
+
dtype: kTensor.dtype,
|
|
604
|
+
});
|
|
605
|
+
}
|
|
447
606
|
}
|
|
448
607
|
|
|
449
608
|
if (normed !== attentionInput) releaseBuffer(normed.buffer);
|
|
@@ -451,24 +610,32 @@ export async function runLayerAttentionGPU(
|
|
|
451
610
|
|
|
452
611
|
// 3. RoPE (modifies tensor in-place)
|
|
453
612
|
|
|
454
|
-
if (!disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
|
|
455
|
-
|
|
456
|
-
numHeads,
|
|
613
|
+
if (!ropeApplied && !disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
|
|
614
|
+
const ropeOptions = {
|
|
457
615
|
headDim,
|
|
458
616
|
rotaryDim: config.ropeRotaryDim,
|
|
617
|
+
pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
|
|
459
618
|
interleaved: config.ropeInterleaved,
|
|
460
619
|
startPos: currentSeqLen,
|
|
461
620
|
executionPolicies: state.executionPolicies ?? null,
|
|
462
|
-
}
|
|
463
|
-
if (
|
|
464
|
-
await
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
621
|
+
};
|
|
622
|
+
if (canUseRoPEQK(qTensor, kTensor, { reusesSharedKV })) {
|
|
623
|
+
await runRoPEQK(qTensor, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
|
|
624
|
+
...ropeOptions,
|
|
625
|
+
numQHeads: numHeads,
|
|
626
|
+
numKVHeads,
|
|
627
|
+
});
|
|
628
|
+
} else {
|
|
629
|
+
await runRoPE(qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
|
|
630
|
+
...ropeOptions,
|
|
631
|
+
numHeads,
|
|
471
632
|
});
|
|
633
|
+
if (!reusesSharedKV) {
|
|
634
|
+
await runRoPE(kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
|
|
635
|
+
...ropeOptions,
|
|
636
|
+
numHeads: numKVHeads,
|
|
637
|
+
});
|
|
638
|
+
}
|
|
472
639
|
}
|
|
473
640
|
|
|
474
641
|
// Trace RoPE outputs
|
|
@@ -529,7 +696,7 @@ export async function runLayerAttentionGPU(
|
|
|
529
696
|
|
|
530
697
|
// 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
|
|
531
698
|
|
|
532
|
-
if (state.kvCache?.hasGPUCache?.()) {
|
|
699
|
+
if (!diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
|
|
533
700
|
if (state.kvCache.kvDtype === 'f16') {
|
|
534
701
|
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
535
702
|
if (kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
|
|
@@ -557,11 +724,66 @@ export async function runLayerAttentionGPU(
|
|
|
557
724
|
}
|
|
558
725
|
|
|
559
726
|
// Resolve KV cache state and build dispatch parameters (shared with record.js)
|
|
560
|
-
|
|
727
|
+
let kvState;
|
|
728
|
+
if (diffusionGemmaDecoder) {
|
|
729
|
+
const decoderKVDtype = kvCacheDtype ?? kTensor.dtype;
|
|
730
|
+
let decoderKTensor = kTensor;
|
|
731
|
+
let decoderVTensor = vTensor;
|
|
732
|
+
if (decoderKVDtype === 'f16' && decoderKTensor.dtype !== 'f16') {
|
|
733
|
+
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
734
|
+
if (!hasExplicitF16KvContract) {
|
|
735
|
+
assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f16', 'DiffusionGemma decoder K would be narrowed implicitly.');
|
|
736
|
+
}
|
|
737
|
+
decoderKTensor = await castF32ToF16(decoderKTensor);
|
|
738
|
+
decoderKTemp = decoderKTensor;
|
|
739
|
+
} else if (decoderKVDtype === 'f32' && decoderKTensor.dtype !== 'f32') {
|
|
740
|
+
assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f32', 'DiffusionGemma decoder K would be widened implicitly.');
|
|
741
|
+
decoderKTensor = await castF16ToF32(decoderKTensor);
|
|
742
|
+
decoderKTemp = decoderKTensor;
|
|
743
|
+
}
|
|
744
|
+
if (decoderKVDtype === 'f16' && decoderVTensor.dtype !== 'f16') {
|
|
745
|
+
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
746
|
+
if (!hasExplicitF16KvContract) {
|
|
747
|
+
assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f16', 'DiffusionGemma decoder V would be narrowed implicitly.');
|
|
748
|
+
}
|
|
749
|
+
decoderVTensor = await castF32ToF16(decoderVTensor);
|
|
750
|
+
decoderVTemp = decoderVTensor;
|
|
751
|
+
} else if (decoderKVDtype === 'f32' && decoderVTensor.dtype !== 'f32') {
|
|
752
|
+
assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f32', 'DiffusionGemma decoder V would be widened implicitly.');
|
|
753
|
+
decoderVTensor = await castF16ToF32(decoderVTensor);
|
|
754
|
+
decoderVTemp = decoderVTensor;
|
|
755
|
+
}
|
|
756
|
+
kvState = await createDiffusionGemmaDecoderKVState({
|
|
757
|
+
state,
|
|
758
|
+
layerIdx,
|
|
759
|
+
kTensor: decoderKTensor,
|
|
760
|
+
vTensor: decoderVTensor,
|
|
761
|
+
currentSeqLen,
|
|
762
|
+
numTokens,
|
|
763
|
+
numKVHeads,
|
|
764
|
+
headDim,
|
|
765
|
+
layerType,
|
|
766
|
+
slidingWindow,
|
|
767
|
+
kvDtype: decoderKVDtype,
|
|
768
|
+
});
|
|
769
|
+
if (decoderKTemp) {
|
|
770
|
+
releaseBuffer(decoderKTemp.buffer);
|
|
771
|
+
decoderKTemp = null;
|
|
772
|
+
}
|
|
773
|
+
if (decoderVTemp && decoderVTemp.buffer !== decoderKTensor.buffer) {
|
|
774
|
+
releaseBuffer(decoderVTemp.buffer);
|
|
775
|
+
}
|
|
776
|
+
decoderVTemp = null;
|
|
777
|
+
} else {
|
|
778
|
+
kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens);
|
|
779
|
+
}
|
|
561
780
|
const dispatchConfig = {
|
|
562
781
|
layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
|
|
563
|
-
slidingWindow
|
|
564
|
-
|
|
782
|
+
slidingWindow: diffusionGemmaDecoder ? null : slidingWindow,
|
|
783
|
+
layerType,
|
|
784
|
+
layerTypes: config.layerTypes,
|
|
785
|
+
queryPreAttnScalar,
|
|
786
|
+
causalAttention: diffusionGemmaDecoder ? false : config.causalAttention,
|
|
565
787
|
activationDtype: attentionActivationDtype,
|
|
566
788
|
kvCacheDtype: attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? null,
|
|
567
789
|
};
|
|
@@ -763,6 +985,15 @@ export async function runLayerAttentionGPU(
|
|
|
763
985
|
kForAttn = createTensor(kCasted.buffer, kCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_K');
|
|
764
986
|
vForAttn = createTensor(vCasted.buffer, vCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_V');
|
|
765
987
|
}
|
|
988
|
+
// Session precedence is runtime-over-manifest per config-style-guide
|
|
989
|
+
// §Category Rules: `resolvedSession = merge(manifest.session, runtime.session)`
|
|
990
|
+
// with runtime fields winning. getRuntimeConfig() returns that merged
|
|
991
|
+
// session (state.runtimeConfig.inference.session is a stale snapshot —
|
|
992
|
+
// see comment above at the rmsNorm fusion check). The kernel itself
|
|
993
|
+
// enforces remaining preconditions (head_dim=256, etc.).
|
|
994
|
+
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
995
|
+
const useFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useFlashPrefillAttention === true && numTokens > 1;
|
|
996
|
+
const useOrtFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useOrtFlashPrefillAttention === true && numTokens > 1;
|
|
766
997
|
const result = await runAttention(qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
|
|
767
998
|
seqLen: numTokens,
|
|
768
999
|
kvLen: kvState.kvLenForAttention,
|
|
@@ -780,6 +1011,8 @@ export async function runLayerAttentionGPU(
|
|
|
780
1011
|
kvPageTable: kvState.kvPageTable,
|
|
781
1012
|
kvPageSize: kvState.kvPageSize,
|
|
782
1013
|
kernelPath,
|
|
1014
|
+
useFlashPrefill,
|
|
1015
|
+
useOrtFlashPrefill,
|
|
783
1016
|
});
|
|
784
1017
|
if (prefillFallbackNeedsCast) {
|
|
785
1018
|
if (kTensor.dtype !== 'f16') releaseBuffer(kForAttn.buffer);
|
|
@@ -794,7 +1027,24 @@ export async function runLayerAttentionGPU(
|
|
|
794
1027
|
throw new Error(`Unsupported attention kernel variant "${attentionKernelVariant}" at layer ${layerIdx}`);
|
|
795
1028
|
}
|
|
796
1029
|
|
|
797
|
-
|
|
1030
|
+
try {
|
|
1031
|
+
attnOutput = await runAttentionKernel();
|
|
1032
|
+
} finally {
|
|
1033
|
+
if (decoderKVState?.ownedBuffers) {
|
|
1034
|
+
for (const buffer of decoderKVState.ownedBuffers) {
|
|
1035
|
+
releaseBuffer(buffer);
|
|
1036
|
+
}
|
|
1037
|
+
decoderKVState.ownedBuffers = null;
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
await runProbes('attn_core_out', attnOutput.buffer, {
|
|
1041
|
+
layerIdx,
|
|
1042
|
+
numTokens,
|
|
1043
|
+
hiddenSize: numHeads * headDim,
|
|
1044
|
+
probes: state.debugProbes,
|
|
1045
|
+
operatorDiagnostics: state.operatorDiagnostics,
|
|
1046
|
+
dtype: attnOutput.dtype,
|
|
1047
|
+
});
|
|
798
1048
|
|
|
799
1049
|
// Trace attention output
|
|
800
1050
|
if (kernelTrace.enabled) {
|
|
@@ -819,10 +1069,27 @@ export async function runLayerAttentionGPU(
|
|
|
819
1069
|
|
|
820
1070
|
attnForProjection = attnOutput;
|
|
821
1071
|
if (qGateTensor) {
|
|
1072
|
+
// The Qwen3_5 reference (HF transformers) full-attention path
|
|
1073
|
+
// applies sigmoid(gate) — NOT silu(gate) — to the attention
|
|
1074
|
+
// output regardless of how the architecture's gate label is
|
|
1075
|
+
// surfaced in the HF config. See Qwen3_5Attention.forward in
|
|
1076
|
+
// transformers/models/qwen3_5/modeling_qwen3_5.py:
|
|
1077
|
+
// attn_output = attn_output * torch.sigmoid(gate)
|
|
1078
|
+
// The earlier `swish` -> `silu` mapping here was incorrect and
|
|
1079
|
+
// produced sign-flipped output for any token whose gate value
|
|
1080
|
+
// was negative (silu is sign-preserving via x * sigmoid(x);
|
|
1081
|
+
// sigmoid is unconditionally in (0, 1)). At Qwen 3.6 27B scale
|
|
1082
|
+
// this drove residual-stream divergence by L=3 (HF reference vs
|
|
1083
|
+
// upstream rel_l2≈1.37, mean per-token correlation ≈ -0.96 — a
|
|
1084
|
+
// sign-flip pattern). config.outputGateType is retained for
|
|
1085
|
+
// future architectures whose gate truly is silu, but Qwen 3.6's
|
|
1086
|
+
// `output_gate_type=swish` still routes here as sigmoid because
|
|
1087
|
+
// that is what the reference forward computes.
|
|
1088
|
+
const gateActivation = 'sigmoid';
|
|
822
1089
|
attnForProjection = await runSiLU(attnOutput, {
|
|
823
1090
|
size: numTokens * numHeads * headDim,
|
|
824
1091
|
gate: qGateTensor,
|
|
825
|
-
gateActivation
|
|
1092
|
+
gateActivation,
|
|
826
1093
|
inputActivation: 'identity',
|
|
827
1094
|
swigluLimit: null,
|
|
828
1095
|
});
|
|
@@ -858,6 +1125,7 @@ export async function runLayerAttentionGPU(
|
|
|
858
1125
|
// Use fused o_proj + residual for decode when possible
|
|
859
1126
|
// Note: dtype from WeightBuffer metadata (buffer-dtypes WeakMap removed)
|
|
860
1127
|
const oProjDtype = getWeightDtype(oProjBuf);
|
|
1128
|
+
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
861
1129
|
const canUseFused = selectRuleValue('inference', 'attention', 'useFusedOProjResidual', {
|
|
862
1130
|
allowFusedResidual: shouldUseFusedMatmulResidual(numTokens),
|
|
863
1131
|
hasResidual: Boolean(residualTensor),
|
|
@@ -867,7 +1135,16 @@ export async function runLayerAttentionGPU(
|
|
|
867
1135
|
hasLoRA: Boolean(loraO),
|
|
868
1136
|
oProjIsF16: oProjDtype === 'f16',
|
|
869
1137
|
}); // GEMV kernel expects f16 weights
|
|
870
|
-
|
|
1138
|
+
const canUseWideTileResidual = canUseQ4KWideTileResidualFusion({
|
|
1139
|
+
phase,
|
|
1140
|
+
session: mergedSession,
|
|
1141
|
+
hasResidual: Boolean(residualTensor),
|
|
1142
|
+
residualMatches: Boolean(residualTensor && residualTensor.dtype === oProjInput.dtype),
|
|
1143
|
+
inputDtype: oProjInput.dtype,
|
|
1144
|
+
outputDtype: oProjOutputDtype,
|
|
1145
|
+
oProjDtype,
|
|
1146
|
+
hasLoRA: Boolean(loraO),
|
|
1147
|
+
});
|
|
871
1148
|
if (canUseFused && residualTensor) {
|
|
872
1149
|
// FUSED PATH: o_proj matmul + residual add in one dispatch
|
|
873
1150
|
output = await runMatmulResidualFused(oProjInput, oProjBuf, residualTensor, {
|
|
@@ -880,7 +1157,7 @@ export async function runLayerAttentionGPU(
|
|
|
880
1157
|
trace.attn(layerIdx, `Using fused o_proj+residual path`);
|
|
881
1158
|
}
|
|
882
1159
|
} else {
|
|
883
|
-
// STANDARD PATH: o_proj matmul only
|
|
1160
|
+
// STANDARD PATH: o_proj matmul only unless Q4K WideTile fuses residual in the epilogue
|
|
884
1161
|
output = await runMatmul(oProjInput, oProjBuf, numTokens, hiddenSize, numHeads * headDim, {
|
|
885
1162
|
transposeB: 'auto',
|
|
886
1163
|
role: 'o_proj',
|
|
@@ -888,7 +1165,9 @@ export async function runLayerAttentionGPU(
|
|
|
888
1165
|
kernelPath,
|
|
889
1166
|
outputDtype: oProjOutputDtype,
|
|
890
1167
|
executionPolicies: state.executionPolicies ?? null,
|
|
1168
|
+
residualTensor: canUseWideTileResidual ? residualTensor : null,
|
|
891
1169
|
});
|
|
1170
|
+
residualFused = canUseWideTileResidual;
|
|
892
1171
|
}
|
|
893
1172
|
// Release temporary buffer if we created it (original was not already on GPU)
|
|
894
1173
|
if (!isGpuBufferInstance(layerWeights.oProj) && !isWeightBuffer(layerWeights.oProj)) {
|
|
@@ -973,7 +1252,9 @@ export async function runLayerAttentionGPU(
|
|
|
973
1252
|
}
|
|
974
1253
|
if (!reusesSharedKV && !storeSharedKV) {
|
|
975
1254
|
releaseBuffer(kTensor.buffer);
|
|
976
|
-
|
|
1255
|
+
if (vTensor.buffer !== kTensor.buffer) {
|
|
1256
|
+
releaseBuffer(vTensor.buffer);
|
|
1257
|
+
}
|
|
977
1258
|
}
|
|
978
1259
|
for (const buffer of buffersToRelease) {
|
|
979
1260
|
releaseBuffer(buffer);
|
|
@@ -1020,6 +1301,18 @@ export async function runLayerAttentionGPU(
|
|
|
1020
1301
|
if (attentionInputTemp && attentionInput?.buffer) {
|
|
1021
1302
|
releaseOnce(attentionInput.buffer);
|
|
1022
1303
|
}
|
|
1304
|
+
if (decoderKTemp?.buffer) {
|
|
1305
|
+
releaseOnce(decoderKTemp.buffer);
|
|
1306
|
+
}
|
|
1307
|
+
if (decoderVTemp?.buffer) {
|
|
1308
|
+
releaseOnce(decoderVTemp.buffer);
|
|
1309
|
+
}
|
|
1310
|
+
if (decoderKVState?.ownedBuffers) {
|
|
1311
|
+
for (const buffer of decoderKVState.ownedBuffers) {
|
|
1312
|
+
releaseOnce(buffer);
|
|
1313
|
+
}
|
|
1314
|
+
decoderKVState.ownedBuffers = null;
|
|
1315
|
+
}
|
|
1023
1316
|
throw error;
|
|
1024
1317
|
}
|
|
1025
1318
|
}
|
|
@@ -39,8 +39,12 @@ export interface AttentionConfig {
|
|
|
39
39
|
attnSoftcap?: number;
|
|
40
40
|
/** Gemma 2 attention scaling: uses head_dim (256) instead of sqrt(head_dim) (16). */
|
|
41
41
|
queryPreAttnScalar?: number;
|
|
42
|
-
/** Apply query/key RMSNorm
|
|
42
|
+
/** Apply query/key RMSNorm where per-head weights are declared. */
|
|
43
43
|
queryKeyNorm?: boolean;
|
|
44
|
+
/** Layers that apply query/key RMSNorm; null means all layers when queryKeyNorm=true. */
|
|
45
|
+
queryKeyNormLayers?: number[] | null;
|
|
46
|
+
/** Layers that carry explicit Q/K RMSNorm scale tensors; null means missing weights are invalid. */
|
|
47
|
+
queryKeyNormWeightLayers?: number[] | null;
|
|
44
48
|
/** Apply unit-scale value RMSNorm before attention. */
|
|
45
49
|
valueNorm?: boolean;
|
|
46
50
|
/** Apply sigmoid gate from q_proj split to attention output before o_proj. */
|
|
@@ -56,6 +60,8 @@ export interface AttentionConfig {
|
|
|
56
60
|
} | null;
|
|
57
61
|
/** RoPE rotary dimension (may differ from headDim with partial rotary). */
|
|
58
62
|
ropeRotaryDim?: number;
|
|
63
|
+
/** RoPE frequency base dimension; non-interleaved kernels use this as the rotate-half pair span. */
|
|
64
|
+
ropeFrequencyBaseDim?: number;
|
|
59
65
|
/** Whether RoPE uses interleaved layout. */
|
|
60
66
|
ropeInterleaved?: boolean;
|
|
61
67
|
/** Token IDs for the current micro-batch (required by BDPA KV ingestion). */
|
|
@@ -36,18 +36,8 @@ export type ChatTemplateType =
|
|
|
36
36
|
| 'translategemma'
|
|
37
37
|
| null;
|
|
38
38
|
|
|
39
|
-
export declare function formatGemmaChat(messages: ChatMessage[]): string;
|
|
40
|
-
|
|
41
39
|
export interface ChatFormatOptions {
|
|
42
40
|
thinking?: boolean;
|
|
43
41
|
}
|
|
44
42
|
|
|
45
|
-
export declare function formatGemma4Chat(messages: ChatMessage[], options?: ChatFormatOptions): string;
|
|
46
|
-
|
|
47
|
-
export declare function formatLlama3Chat(messages: ChatMessage[]): string;
|
|
48
|
-
|
|
49
|
-
export declare function formatGptOssChat(messages: ChatMessage[]): string;
|
|
50
|
-
|
|
51
|
-
export declare function formatTranslateGemmaChat(messages: ChatMessage[]): string;
|
|
52
|
-
|
|
53
43
|
export declare function formatChatMessages(messages: ChatMessage[], templateType?: ChatTemplateType, options?: ChatFormatOptions): string;
|