doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
3
|
import { isGpuBufferInstance, isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
|
|
4
|
-
import {
|
|
4
|
+
import { getKernelCapabilities } from '../../../../gpu/device.js';
|
|
5
|
+
import { acquireBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
|
|
5
6
|
import {
|
|
6
7
|
recordMatmul,
|
|
7
8
|
recordRMSNorm,
|
|
8
9
|
recordRoPE,
|
|
10
|
+
canUseRoPEQK,
|
|
11
|
+
recordRoPEQK,
|
|
9
12
|
recordAttention,
|
|
10
13
|
recordAttentionTiered,
|
|
11
14
|
recordAttentionTieredQuant,
|
|
@@ -20,7 +23,7 @@ import {
|
|
|
20
23
|
import { createTensor } from '../../../../gpu/tensor.js';
|
|
21
24
|
import { applyLoRA } from '../lora-apply.js';
|
|
22
25
|
import { getLoRAModule } from '../lora.js';
|
|
23
|
-
import { log, trace } from '../../../../debug/index.js';
|
|
26
|
+
import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
|
|
24
27
|
import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
25
28
|
import {
|
|
26
29
|
recordAttentionInputs,
|
|
@@ -29,9 +32,13 @@ import {
|
|
|
29
32
|
projectAttentionQKV,
|
|
30
33
|
applyAttentionQKNorm,
|
|
31
34
|
applyAttentionValueNorm,
|
|
35
|
+
hasAttentionProjectionDiagnostics,
|
|
36
|
+
hasAttentionStageDiagnostics,
|
|
37
|
+
resolveAttentionQKNormState,
|
|
32
38
|
} from './projections.js';
|
|
33
39
|
import { prepareAttentionProjectionInput } from './output-projection.js';
|
|
34
40
|
import { runProbes } from '../probes.js';
|
|
41
|
+
import { decodeReadback, getLogitsHealth } from '../debug-utils/index.js';
|
|
35
42
|
|
|
36
43
|
import { releaseOrTrack, shouldDebugLayer } from './types.js';
|
|
37
44
|
import {
|
|
@@ -40,6 +47,7 @@ import {
|
|
|
40
47
|
} from '../../../../config/kernel-path-loader.js';
|
|
41
48
|
import {
|
|
42
49
|
resolveKVCacheState,
|
|
50
|
+
createDiffusionGemmaDecoderKVState,
|
|
43
51
|
buildAttentionDispatchParams,
|
|
44
52
|
buildAttentionInputsData,
|
|
45
53
|
} from './dispatch-params.js';
|
|
@@ -57,6 +65,93 @@ import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js'
|
|
|
57
65
|
|
|
58
66
|
const ATTENTION_DTYPE_LOGGED = new Set();
|
|
59
67
|
|
|
68
|
+
function isWideTileQ4KPhaseEnabled(session, phase) {
|
|
69
|
+
return phase === 'decode'
|
|
70
|
+
? session?.useWideTileQ4KDecode === true
|
|
71
|
+
: session?.useWideTileQ4KPrefill === true;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function canUseQ4KWideTileResidualFusion(options = {}) {
|
|
75
|
+
return options.hasResidual === true
|
|
76
|
+
&& options.oProjDtype === 'q4k'
|
|
77
|
+
&& options.inputDtype === 'f32'
|
|
78
|
+
&& options.outputDtype === 'f32'
|
|
79
|
+
&& options.residualMatches === true
|
|
80
|
+
&& options.hasLoRA !== true
|
|
81
|
+
&& getKernelCapabilities().hasF16 === true
|
|
82
|
+
&& getKernelCapabilities().hasSubgroups === true
|
|
83
|
+
&& options.session?.useWideTileResidualFusion === true
|
|
84
|
+
&& options.session?.retainQ4KMaterialization === true
|
|
85
|
+
&& isWideTileQ4KPhaseEnabled(options.session, options.phase);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function shouldTraceRecordedHealth(layerIdx, debugFlags) {
|
|
89
|
+
const debugLayers = debugFlags?.debugLayers;
|
|
90
|
+
return isTraceEnabled('logits')
|
|
91
|
+
&& Array.isArray(debugLayers)
|
|
92
|
+
&& shouldDebugLayer(layerIdx, debugLayers);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function enqueueRecordedTensorHealth(recorder, label, tensor, dtype, elementCount) {
|
|
96
|
+
if (!recorder || !tensor?.buffer || !Number.isFinite(elementCount) || elementCount <= 0) {
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
|
|
100
|
+
recorder.enqueueCompletionTask(async () => {
|
|
101
|
+
const data = await readBuffer(tensor.buffer, elementCount * bytesPerElement);
|
|
102
|
+
trace.logits(label, getLogitsHealth(decodeReadback(data, dtype)));
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function resolveDirectF16KVCacheWrite(options) {
|
|
107
|
+
const {
|
|
108
|
+
state,
|
|
109
|
+
layerIdx,
|
|
110
|
+
currentSeqLen,
|
|
111
|
+
numTokens,
|
|
112
|
+
numKVHeads,
|
|
113
|
+
headDim,
|
|
114
|
+
reusesSharedKV,
|
|
115
|
+
storeSharedKV,
|
|
116
|
+
diffusionGemmaDecoder,
|
|
117
|
+
valueNorm,
|
|
118
|
+
diagnosticsEnabled,
|
|
119
|
+
disableRoPE,
|
|
120
|
+
} = options;
|
|
121
|
+
if (
|
|
122
|
+
numTokens !== 1
|
|
123
|
+
|| reusesSharedKV === true
|
|
124
|
+
|| storeSharedKV === true
|
|
125
|
+
|| diffusionGemmaDecoder === true
|
|
126
|
+
|| valueNorm === true
|
|
127
|
+
|| diagnosticsEnabled === true
|
|
128
|
+
|| disableRoPE === true
|
|
129
|
+
) {
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
const cache = state?.kvCache;
|
|
133
|
+
if (
|
|
134
|
+
!cache?.hasGPUCache?.()
|
|
135
|
+
|| cache.kvDtype !== 'f16'
|
|
136
|
+
|| cache.layout !== 'contiguous'
|
|
137
|
+
|| cache.windowSize != null
|
|
138
|
+
|| cache.layerSpecs != null
|
|
139
|
+
|| typeof cache.recordF16UpdateAlreadyWrittenFromGPU !== 'function'
|
|
140
|
+
) {
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
const gpuBuffers = cache.getGPUBuffers(layerIdx);
|
|
144
|
+
const layout = gpuBuffers?.layout ?? cache.layout;
|
|
145
|
+
if (layout !== 'contiguous' || !gpuBuffers?.keysGPU || !gpuBuffers?.valuesGPU) {
|
|
146
|
+
return null;
|
|
147
|
+
}
|
|
148
|
+
return {
|
|
149
|
+
keysBuffer: gpuBuffers.keysGPU,
|
|
150
|
+
valuesBuffer: gpuBuffers.valuesGPU,
|
|
151
|
+
dstOffset: currentSeqLen * numKVHeads * headDim,
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
|
|
60
155
|
function assertAttentionDtypeTransitionAllowed(state, fromDtype, toDtype, detail, transitionDeclaredBy = null) {
|
|
61
156
|
assertImplicitDtypeTransitionAllowed({
|
|
62
157
|
executionPolicies: state?.executionPolicies ?? null,
|
|
@@ -104,6 +199,8 @@ export async function recordLayerAttentionGPU(
|
|
|
104
199
|
multimodalBidirectionalSpan = null,
|
|
105
200
|
sharedKVSourceLayerIdx = null,
|
|
106
201
|
storeSharedKV = false,
|
|
202
|
+
diffusionGemmaDecoder = false,
|
|
203
|
+
precomputedInputNorm = null,
|
|
107
204
|
} = config;
|
|
108
205
|
|
|
109
206
|
const phase = isPrefill ? 'prefill' : 'decode';
|
|
@@ -125,6 +222,9 @@ export async function recordLayerAttentionGPU(
|
|
|
125
222
|
const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: useF16Activations });
|
|
126
223
|
const kvCacheDtype = attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? kvCacheFallback;
|
|
127
224
|
const allowF16Attention = useF16Activations && kvCacheDtype === 'f16';
|
|
225
|
+
const attentionInputTransitionDeclaredBy = attentionPrecisionContract.explicitInputDtype
|
|
226
|
+
? 'step_precision'
|
|
227
|
+
: null;
|
|
128
228
|
let attentionInput = input;
|
|
129
229
|
let attentionInputTemp = false;
|
|
130
230
|
let normed = attentionInput;
|
|
@@ -139,12 +239,16 @@ export async function recordLayerAttentionGPU(
|
|
|
139
239
|
let oProjInputTemp = null;
|
|
140
240
|
let retainSharedKvBuffers = false;
|
|
141
241
|
let valueAliasesKey = false;
|
|
242
|
+
let decoderKVState = null;
|
|
243
|
+
let decoderKTemp = null;
|
|
244
|
+
let decoderVTemp = null;
|
|
142
245
|
if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
|
|
143
246
|
assertAttentionDtypeTransitionAllowed(
|
|
144
247
|
state,
|
|
145
248
|
input.dtype,
|
|
146
249
|
attentionActivationDtype,
|
|
147
|
-
'The attention kernel selection would widen the input implicitly.'
|
|
250
|
+
'The attention kernel selection would widen the input implicitly.',
|
|
251
|
+
attentionInputTransitionDeclaredBy
|
|
148
252
|
);
|
|
149
253
|
attentionInput = attentionActivationDtype === 'f16'
|
|
150
254
|
? await recordCastF32ToF16(recorder, input)
|
|
@@ -188,7 +292,25 @@ export async function recordLayerAttentionGPU(
|
|
|
188
292
|
&& attentionInput.dtype === 'f32';
|
|
189
293
|
|
|
190
294
|
try {
|
|
191
|
-
if (
|
|
295
|
+
if (precomputedInputNorm) {
|
|
296
|
+
if (
|
|
297
|
+
isPrefill
|
|
298
|
+
|| numTokens !== 1
|
|
299
|
+
|| skipInputNorm
|
|
300
|
+
|| !layerWeights.inputNorm
|
|
301
|
+
|| !getNormWeightBuffer
|
|
302
|
+
|| precomputedInputNorm.dtype !== attentionInput.dtype
|
|
303
|
+
|| precomputedInputNorm.dtype !== 'f32'
|
|
304
|
+
) {
|
|
305
|
+
throw new Error(`Layer ${layerIdx} received an incompatible precomputed input norm tensor.`);
|
|
306
|
+
}
|
|
307
|
+
normed = precomputedInputNorm;
|
|
308
|
+
await runProbes('post_input_norm', normed.buffer, {
|
|
309
|
+
layerIdx, numTokens, hiddenSize,
|
|
310
|
+
probes: state.debugProbes, recorder,
|
|
311
|
+
operatorDiagnostics: state.operatorDiagnostics, dtype: normed.dtype,
|
|
312
|
+
});
|
|
313
|
+
} else if (canFuseInputNormProjRec) {
|
|
192
314
|
fusedNormWeightRec = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
193
315
|
fusedNormEpsRec = rmsNormEps;
|
|
194
316
|
fusedNormOffsetRec = config.rmsNormWeightOffset === true;
|
|
@@ -200,6 +322,7 @@ export async function recordLayerAttentionGPU(
|
|
|
200
322
|
batchSize: numTokens,
|
|
201
323
|
hiddenSize,
|
|
202
324
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
325
|
+
label: 'input_norm',
|
|
203
326
|
});
|
|
204
327
|
if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseOrTrack(recorder, normWeightBuf);
|
|
205
328
|
await runProbes('post_input_norm', normed.buffer, {
|
|
@@ -254,7 +377,35 @@ export async function recordLayerAttentionGPU(
|
|
|
254
377
|
}
|
|
255
378
|
const reusesSharedKV = sharedKVEntry != null;
|
|
256
379
|
retainSharedKvBuffers = reusesSharedKV || storeSharedKV;
|
|
257
|
-
|
|
380
|
+
let qkNormApplied = false;
|
|
381
|
+
let ropeApplied = false;
|
|
382
|
+
const qkNormState = resolveAttentionQKNormState({ config, layerWeights, layerIdx, reusesSharedKV });
|
|
383
|
+
const qkNormProjectionDiagnostics = hasAttentionProjectionDiagnostics(state)
|
|
384
|
+
|| shouldTraceRecordedHealth(layerIdx, debugFlags);
|
|
385
|
+
const qkNormRoPEDiagnostics = hasAttentionStageDiagnostics(
|
|
386
|
+
state,
|
|
387
|
+
['q_proj', 'k_proj', 'v_proj', 'q_norm', 'k_norm']
|
|
388
|
+
)
|
|
389
|
+
|| shouldTraceRecordedHealth(layerIdx, debugFlags);
|
|
390
|
+
const runtimeSession = getRuntimeConfig()?.inference?.session;
|
|
391
|
+
const qkNormFusionFlag = runtimeSession?.useFusedQKVSplitQKNorm === true;
|
|
392
|
+
const qkNormRoPEFusionFlag = runtimeSession?.useFusedQKVSplitQKNormRoPE === true;
|
|
393
|
+
const directF16KVCacheWrite = resolveDirectF16KVCacheWrite({
|
|
394
|
+
state,
|
|
395
|
+
layerIdx,
|
|
396
|
+
currentSeqLen,
|
|
397
|
+
numTokens,
|
|
398
|
+
numKVHeads,
|
|
399
|
+
headDim,
|
|
400
|
+
reusesSharedKV,
|
|
401
|
+
storeSharedKV,
|
|
402
|
+
diffusionGemmaDecoder,
|
|
403
|
+
valueNorm: config.valueNorm === true,
|
|
404
|
+
diagnosticsEnabled: qkNormRoPEDiagnostics,
|
|
405
|
+
disableRoPE,
|
|
406
|
+
});
|
|
407
|
+
let kvCacheWriteFused = false;
|
|
408
|
+
({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey, qkNormApplied, ropeApplied, kvCacheWriteFused } = await projectAttentionQKV({
|
|
258
409
|
recorder,
|
|
259
410
|
normed,
|
|
260
411
|
layerWeights,
|
|
@@ -282,16 +433,51 @@ export async function recordLayerAttentionGPU(
|
|
|
282
433
|
fusedNormWeight: fusedNormWeightRec,
|
|
283
434
|
fusedNormEps: fusedNormEpsRec,
|
|
284
435
|
fusedNormOffset: fusedNormOffsetRec,
|
|
436
|
+
qkNormFusion: {
|
|
437
|
+
enabled: qkNormFusionFlag && qkNormState.wantsQKNorm,
|
|
438
|
+
getNormWeightBuffer,
|
|
439
|
+
rmsNormEps,
|
|
440
|
+
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
441
|
+
skipKNorm: qkNormState.skipKNorm,
|
|
442
|
+
allowUnitQKNorm: qkNormState.allowUnitQKNorm,
|
|
443
|
+
projectionDiagnosticsEnabled: qkNormProjectionDiagnostics,
|
|
444
|
+
},
|
|
445
|
+
qkNormRoPEFusion: {
|
|
446
|
+
enabled: qkNormRoPEFusionFlag
|
|
447
|
+
&& qkNormState.wantsQKNorm
|
|
448
|
+
&& !disableRoPE
|
|
449
|
+
&& !!state.ropeFreqsCos
|
|
450
|
+
&& !!state.ropeFreqsSin,
|
|
451
|
+
getNormWeightBuffer,
|
|
452
|
+
rmsNormEps,
|
|
453
|
+
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
454
|
+
skipKNorm: qkNormState.skipKNorm,
|
|
455
|
+
allowUnitQKNorm: qkNormState.allowUnitQKNorm,
|
|
456
|
+
projectionDiagnosticsEnabled: qkNormRoPEDiagnostics,
|
|
457
|
+
freqsCos: state.ropeFreqsCos,
|
|
458
|
+
freqsSin: state.ropeFreqsSin,
|
|
459
|
+
startPos: currentSeqLen,
|
|
460
|
+
rotaryDim: config.ropeRotaryDim,
|
|
461
|
+
pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
|
|
462
|
+
interleaved: config.ropeInterleaved,
|
|
463
|
+
reusesSharedKV,
|
|
464
|
+
f16KVCacheWrite: directF16KVCacheWrite,
|
|
465
|
+
},
|
|
285
466
|
}));
|
|
467
|
+
if (!kvCacheWriteFused && (!kTensor || !vTensor)) {
|
|
468
|
+
throw new Error('Recorded attention projection returned missing K/V tensors without a fused KV cache write.');
|
|
469
|
+
}
|
|
286
470
|
// Deferred release of the norm weight buffer for fused path.
|
|
287
471
|
if (fusedNormWeightRec && fusedNormOwnedRec) {
|
|
288
472
|
releaseOrTrack(recorder, fusedNormWeightRec);
|
|
289
473
|
}
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
474
|
+
if (qTensor) {
|
|
475
|
+
await runProbes('q_proj', qTensor.buffer, {
|
|
476
|
+
layerIdx, numTokens, hiddenSize: numHeads * headDim,
|
|
477
|
+
probes: state.debugProbes, recorder,
|
|
478
|
+
operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
|
|
479
|
+
});
|
|
480
|
+
}
|
|
295
481
|
if (kTensor) {
|
|
296
482
|
await runProbes('k_proj', kTensor.buffer, {
|
|
297
483
|
layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
|
|
@@ -306,43 +492,83 @@ export async function recordLayerAttentionGPU(
|
|
|
306
492
|
operatorDiagnostics: state.operatorDiagnostics, dtype: vTensor.dtype,
|
|
307
493
|
});
|
|
308
494
|
}
|
|
495
|
+
if (qTensor && shouldTraceRecordedHealth(layerIdx, debugFlags)) {
|
|
496
|
+
enqueueRecordedTensorHealth(
|
|
497
|
+
recorder,
|
|
498
|
+
`L${layerIdx}.q_proj_HEALTH`,
|
|
499
|
+
qTensor,
|
|
500
|
+
qTensor.dtype,
|
|
501
|
+
numTokens * numHeads * headDim
|
|
502
|
+
);
|
|
503
|
+
enqueueRecordedTensorHealth(
|
|
504
|
+
recorder,
|
|
505
|
+
`L${layerIdx}.k_proj_HEALTH`,
|
|
506
|
+
kTensor,
|
|
507
|
+
kTensor?.dtype ?? null,
|
|
508
|
+
numTokens * numKVHeads * headDim
|
|
509
|
+
);
|
|
510
|
+
enqueueRecordedTensorHealth(
|
|
511
|
+
recorder,
|
|
512
|
+
`L${layerIdx}.v_proj_HEALTH`,
|
|
513
|
+
vTensor,
|
|
514
|
+
vTensor?.dtype ?? null,
|
|
515
|
+
numTokens * numKVHeads * headDim
|
|
516
|
+
);
|
|
517
|
+
}
|
|
309
518
|
|
|
310
519
|
// Optional per-head Q/K normalization.
|
|
311
520
|
// Some models use RMSNorm with (1+weight) offset formula, controlled by rmsNormWeightOffset.
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
521
|
+
if (qkNormState.wantsQKNorm && !qkNormApplied) {
|
|
522
|
+
({ qTensor, kTensor } = await applyAttentionQKNorm({
|
|
523
|
+
recorder,
|
|
524
|
+
qTensor,
|
|
525
|
+
kTensor,
|
|
526
|
+
layerWeights,
|
|
527
|
+
getNormWeightBuffer,
|
|
528
|
+
rmsNormEps,
|
|
529
|
+
numTokens,
|
|
530
|
+
numHeads,
|
|
531
|
+
numKVHeads,
|
|
532
|
+
headDim,
|
|
533
|
+
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
534
|
+
releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
|
|
535
|
+
skipKNorm: qkNormState.skipKNorm,
|
|
536
|
+
retainKInput: valueAliasesKey,
|
|
537
|
+
allowUnitQKNorm: qkNormState.allowUnitQKNorm,
|
|
538
|
+
}));
|
|
315
539
|
}
|
|
316
|
-
(
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
kTensor,
|
|
320
|
-
layerWeights,
|
|
321
|
-
getNormWeightBuffer,
|
|
322
|
-
rmsNormEps,
|
|
323
|
-
numTokens,
|
|
324
|
-
numHeads,
|
|
325
|
-
numKVHeads,
|
|
326
|
-
headDim,
|
|
327
|
-
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
328
|
-
releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
|
|
329
|
-
skipKNorm: reusesSharedKV,
|
|
330
|
-
retainKInput: valueAliasesKey,
|
|
331
|
-
}));
|
|
332
|
-
await runProbes('q_norm', qTensor.buffer, {
|
|
333
|
-
layerIdx, numTokens, hiddenSize: numHeads * headDim,
|
|
334
|
-
probes: state.debugProbes, recorder,
|
|
335
|
-
operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
|
|
336
|
-
});
|
|
337
|
-
if (kTensor) {
|
|
338
|
-
await runProbes('k_norm', kTensor.buffer, {
|
|
339
|
-
layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
|
|
540
|
+
if (qkNormState.wantsQKNorm) {
|
|
541
|
+
await runProbes('q_norm', qTensor.buffer, {
|
|
542
|
+
layerIdx, numTokens, hiddenSize: numHeads * headDim,
|
|
340
543
|
probes: state.debugProbes, recorder,
|
|
341
|
-
operatorDiagnostics: state.operatorDiagnostics, dtype:
|
|
544
|
+
operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
|
|
342
545
|
});
|
|
546
|
+
if (kTensor) {
|
|
547
|
+
await runProbes('k_norm', kTensor.buffer, {
|
|
548
|
+
layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
|
|
549
|
+
probes: state.debugProbes, recorder,
|
|
550
|
+
operatorDiagnostics: state.operatorDiagnostics, dtype: kTensor.dtype,
|
|
551
|
+
});
|
|
552
|
+
}
|
|
553
|
+
if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
|
|
554
|
+
enqueueRecordedTensorHealth(
|
|
555
|
+
recorder,
|
|
556
|
+
`L${layerIdx}.q_norm_HEALTH`,
|
|
557
|
+
qTensor,
|
|
558
|
+
qTensor.dtype,
|
|
559
|
+
numTokens * numHeads * headDim
|
|
560
|
+
);
|
|
561
|
+
enqueueRecordedTensorHealth(
|
|
562
|
+
recorder,
|
|
563
|
+
`L${layerIdx}.k_norm_HEALTH`,
|
|
564
|
+
kTensor,
|
|
565
|
+
kTensor?.dtype ?? null,
|
|
566
|
+
numTokens * numKVHeads * headDim
|
|
567
|
+
);
|
|
568
|
+
}
|
|
343
569
|
}
|
|
344
570
|
|
|
345
|
-
if (config.valueNorm === true && !reusesSharedKV) {
|
|
571
|
+
if (!kvCacheWriteFused && config.valueNorm === true && !reusesSharedKV) {
|
|
346
572
|
const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
|
|
347
573
|
vTensor = await applyAttentionValueNorm({
|
|
348
574
|
recorder,
|
|
@@ -368,28 +594,50 @@ export async function recordLayerAttentionGPU(
|
|
|
368
594
|
if (attentionInputTemp) recorder.trackTemporaryBuffer(attentionInput.buffer);
|
|
369
595
|
|
|
370
596
|
// 3. RoPE (modifies tensor in-place)
|
|
371
|
-
if (!disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
|
|
372
|
-
|
|
373
|
-
numHeads,
|
|
597
|
+
if (!ropeApplied && !disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
|
|
598
|
+
const ropeOptions = {
|
|
374
599
|
headDim,
|
|
375
600
|
rotaryDim: config.ropeRotaryDim,
|
|
376
601
|
pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
|
|
377
602
|
interleaved: config.ropeInterleaved,
|
|
378
603
|
startPos: currentSeqLen,
|
|
379
604
|
executionPolicies: state.executionPolicies ?? null,
|
|
380
|
-
}
|
|
381
|
-
if (
|
|
382
|
-
await
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
605
|
+
};
|
|
606
|
+
if (canUseRoPEQK(qTensor, kTensor, { reusesSharedKV })) {
|
|
607
|
+
await recordRoPEQK(recorder, qTensor, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
|
|
608
|
+
...ropeOptions,
|
|
609
|
+
numQHeads: numHeads,
|
|
610
|
+
numKVHeads,
|
|
611
|
+
});
|
|
612
|
+
} else {
|
|
613
|
+
await recordRoPE(recorder, qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
|
|
614
|
+
...ropeOptions,
|
|
615
|
+
numHeads,
|
|
390
616
|
});
|
|
617
|
+
if (!reusesSharedKV) {
|
|
618
|
+
await recordRoPE(recorder, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
|
|
619
|
+
...ropeOptions,
|
|
620
|
+
numHeads: numKVHeads,
|
|
621
|
+
});
|
|
622
|
+
}
|
|
391
623
|
}
|
|
392
624
|
}
|
|
625
|
+
if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
|
|
626
|
+
enqueueRecordedTensorHealth(
|
|
627
|
+
recorder,
|
|
628
|
+
`L${layerIdx}.q_rope_HEALTH`,
|
|
629
|
+
qTensor,
|
|
630
|
+
qTensor.dtype,
|
|
631
|
+
numTokens * numHeads * headDim
|
|
632
|
+
);
|
|
633
|
+
enqueueRecordedTensorHealth(
|
|
634
|
+
recorder,
|
|
635
|
+
`L${layerIdx}.k_rope_HEALTH`,
|
|
636
|
+
kTensor,
|
|
637
|
+
kTensor?.dtype ?? null,
|
|
638
|
+
numTokens * numKVHeads * headDim
|
|
639
|
+
);
|
|
640
|
+
}
|
|
393
641
|
|
|
394
642
|
if (storeSharedKV && state.sharedAttentionState) {
|
|
395
643
|
state.sharedAttentionState.set(layerIdx, {
|
|
@@ -401,10 +649,10 @@ export async function recordLayerAttentionGPU(
|
|
|
401
649
|
}
|
|
402
650
|
|
|
403
651
|
// 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
|
|
404
|
-
if (state.kvCache?.hasGPUCache?.()) {
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
if (state.kvCache.kvDtype === 'f16') {
|
|
652
|
+
if (!diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
|
|
653
|
+
if (kvCacheWriteFused) {
|
|
654
|
+
state.kvCache.recordF16UpdateAlreadyWrittenFromGPU(layerIdx, currentSeqLen, numTokens, tokenIds);
|
|
655
|
+
} else if (state.kvCache.kvDtype === 'f16') {
|
|
408
656
|
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
409
657
|
if (kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
|
|
410
658
|
assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'K would be narrowed implicitly for KV cache storage.');
|
|
@@ -412,24 +660,95 @@ export async function recordLayerAttentionGPU(
|
|
|
412
660
|
if (vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
|
|
413
661
|
assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'V would be narrowed implicitly for KV cache storage.');
|
|
414
662
|
}
|
|
415
|
-
const
|
|
416
|
-
|
|
663
|
+
const canWriteDirectF32ToF16 = kTensor.dtype === 'f32'
|
|
664
|
+
&& vTensor.dtype === 'f32'
|
|
665
|
+
&& typeof state.kvCache.recordUpdateF32ToF16FromGPU === 'function';
|
|
666
|
+
if (canWriteDirectF32ToF16) {
|
|
667
|
+
await state.kvCache.recordUpdateF32ToF16FromGPU(
|
|
668
|
+
recorder,
|
|
669
|
+
layerIdx,
|
|
670
|
+
kTensor.buffer,
|
|
671
|
+
vTensor.buffer,
|
|
672
|
+
currentSeqLen,
|
|
673
|
+
numTokens,
|
|
674
|
+
tokenIds
|
|
675
|
+
);
|
|
676
|
+
} else {
|
|
677
|
+
const kCasted = kTensor.dtype === 'f16' ? kTensor : await recordCastF32ToF16(recorder, kTensor);
|
|
678
|
+
const vCasted = vTensor.dtype === 'f16' ? vTensor : await recordCastF32ToF16(recorder, vTensor);
|
|
417
679
|
|
|
418
|
-
|
|
680
|
+
await state.kvCache.recordUpdateFromGPU(recorder, layerIdx, kCasted.buffer, vCasted.buffer, currentSeqLen, numTokens, tokenIds);
|
|
419
681
|
|
|
420
|
-
|
|
421
|
-
|
|
682
|
+
if (kTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(kCasted.buffer);
|
|
683
|
+
if (vTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(vCasted.buffer);
|
|
684
|
+
}
|
|
422
685
|
} else {
|
|
423
686
|
await state.kvCache.recordUpdateFromGPU(recorder, layerIdx, kTensor.buffer, vTensor.buffer, currentSeqLen, numTokens, tokenIds);
|
|
424
687
|
}
|
|
425
688
|
}
|
|
426
689
|
|
|
427
690
|
// Resolve KV cache state and build dispatch parameters (shared with run.js)
|
|
428
|
-
|
|
691
|
+
let kvState;
|
|
692
|
+
if (diffusionGemmaDecoder) {
|
|
693
|
+
const decoderKVDtype = kvCacheDtype ?? kTensor.dtype;
|
|
694
|
+
let decoderKTensor = kTensor;
|
|
695
|
+
let decoderVTensor = vTensor;
|
|
696
|
+
if (decoderKVDtype === 'f16' && decoderKTensor.dtype !== 'f16') {
|
|
697
|
+
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
698
|
+
if (!hasExplicitF16KvContract) {
|
|
699
|
+
assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f16', 'DiffusionGemma decoder K would be narrowed implicitly.');
|
|
700
|
+
}
|
|
701
|
+
decoderKTensor = await recordCastF32ToF16(recorder, decoderKTensor);
|
|
702
|
+
decoderKTemp = decoderKTensor;
|
|
703
|
+
} else if (decoderKVDtype === 'f32' && decoderKTensor.dtype !== 'f32') {
|
|
704
|
+
assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f32', 'DiffusionGemma decoder K would be widened implicitly.');
|
|
705
|
+
decoderKTensor = await recordCastF16ToF32(recorder, decoderKTensor);
|
|
706
|
+
decoderKTemp = decoderKTensor;
|
|
707
|
+
}
|
|
708
|
+
if (decoderKVDtype === 'f16' && decoderVTensor.dtype !== 'f16') {
|
|
709
|
+
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
710
|
+
if (!hasExplicitF16KvContract) {
|
|
711
|
+
assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f16', 'DiffusionGemma decoder V would be narrowed implicitly.');
|
|
712
|
+
}
|
|
713
|
+
decoderVTensor = await recordCastF32ToF16(recorder, decoderVTensor);
|
|
714
|
+
decoderVTemp = decoderVTensor;
|
|
715
|
+
} else if (decoderKVDtype === 'f32' && decoderVTensor.dtype !== 'f32') {
|
|
716
|
+
assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f32', 'DiffusionGemma decoder V would be widened implicitly.');
|
|
717
|
+
decoderVTensor = await recordCastF16ToF32(recorder, decoderVTensor);
|
|
718
|
+
decoderVTemp = decoderVTensor;
|
|
719
|
+
}
|
|
720
|
+
kvState = await createDiffusionGemmaDecoderKVState({
|
|
721
|
+
state,
|
|
722
|
+
layerIdx,
|
|
723
|
+
kTensor: decoderKTensor,
|
|
724
|
+
vTensor: decoderVTensor,
|
|
725
|
+
currentSeqLen,
|
|
726
|
+
numTokens,
|
|
727
|
+
numKVHeads,
|
|
728
|
+
headDim,
|
|
729
|
+
layerType,
|
|
730
|
+
slidingWindow,
|
|
731
|
+
kvDtype: decoderKVDtype,
|
|
732
|
+
recorder,
|
|
733
|
+
});
|
|
734
|
+
if (decoderKTemp) {
|
|
735
|
+
recorder.trackTemporaryBuffer(decoderKTemp.buffer);
|
|
736
|
+
decoderKTemp = null;
|
|
737
|
+
}
|
|
738
|
+
if (decoderVTemp) {
|
|
739
|
+
recorder.trackTemporaryBuffer(decoderVTemp.buffer);
|
|
740
|
+
decoderVTemp = null;
|
|
741
|
+
}
|
|
742
|
+
decoderKVState = kvState;
|
|
743
|
+
} else {
|
|
744
|
+
kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens);
|
|
745
|
+
}
|
|
429
746
|
const dispatchConfig = {
|
|
430
747
|
layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
|
|
431
|
-
slidingWindow
|
|
432
|
-
|
|
748
|
+
slidingWindow: diffusionGemmaDecoder ? null : slidingWindow,
|
|
749
|
+
layerType, layerTypes: config.layerTypes,
|
|
750
|
+
queryPreAttnScalar,
|
|
751
|
+
causalAttention: diffusionGemmaDecoder ? false : config.causalAttention,
|
|
433
752
|
activationDtype: attentionActivationDtype,
|
|
434
753
|
kvCacheDtype: attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? null,
|
|
435
754
|
};
|
|
@@ -447,6 +766,7 @@ export async function recordLayerAttentionGPU(
|
|
|
447
766
|
{ useF16Activations, matmulOutputDtype },
|
|
448
767
|
usedFusedQKV, qTensor, kTensor, vTensor,
|
|
449
768
|
));
|
|
769
|
+
const mergedSessionRec = getRuntimeConfig()?.inference?.session;
|
|
450
770
|
|
|
451
771
|
const attentionKernelRunners = {
|
|
452
772
|
bdpa: async () => {
|
|
@@ -601,9 +921,8 @@ export async function recordLayerAttentionGPU(
|
|
|
601
921
|
// Kernel enforces head_dim=256, f16 KV, contiguous layout; only applies
|
|
602
922
|
// when numTokens > 1 (prefill). Same flag semantics as the non-recorder
|
|
603
923
|
// path in ./run.js.
|
|
604
|
-
const
|
|
605
|
-
const
|
|
606
|
-
const useOrtFlashPrefillRec = mergedSessionRec?.useOrtFlashPrefillAttention === true && numTokens > 1;
|
|
924
|
+
const useFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useFlashPrefillAttention === true && numTokens > 1;
|
|
925
|
+
const useOrtFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useOrtFlashPrefillAttention === true && numTokens > 1;
|
|
607
926
|
return recordAttention(recorder, qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
|
|
608
927
|
seqLen: numTokens,
|
|
609
928
|
kvLen: kvState.kvLenForAttention,
|
|
@@ -631,7 +950,16 @@ export async function recordLayerAttentionGPU(
|
|
|
631
950
|
throw new Error(`Unsupported attention kernel variant "${attentionKernelVariant}" at layer ${layerIdx}`);
|
|
632
951
|
}
|
|
633
952
|
|
|
634
|
-
|
|
953
|
+
try {
|
|
954
|
+
attnOutput = await runAttentionKernel();
|
|
955
|
+
} finally {
|
|
956
|
+
if (decoderKVState?.ownedBuffers) {
|
|
957
|
+
for (const buffer of decoderKVState.ownedBuffers) {
|
|
958
|
+
recorder.trackTemporaryBuffer(buffer);
|
|
959
|
+
}
|
|
960
|
+
decoderKVState.ownedBuffers = null;
|
|
961
|
+
}
|
|
962
|
+
}
|
|
635
963
|
await runProbes('attn_core_out', attnOutput.buffer, {
|
|
636
964
|
layerIdx,
|
|
637
965
|
numTokens,
|
|
@@ -641,6 +969,15 @@ export async function recordLayerAttentionGPU(
|
|
|
641
969
|
operatorDiagnostics: state.operatorDiagnostics,
|
|
642
970
|
dtype: attnOutput.dtype,
|
|
643
971
|
});
|
|
972
|
+
if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
|
|
973
|
+
enqueueRecordedTensorHealth(
|
|
974
|
+
recorder,
|
|
975
|
+
`L${layerIdx}.attn_core_out_HEALTH`,
|
|
976
|
+
attnOutput,
|
|
977
|
+
attnOutput.dtype,
|
|
978
|
+
numTokens * numHeads * headDim
|
|
979
|
+
);
|
|
980
|
+
}
|
|
644
981
|
|
|
645
982
|
attnForProjection = attnOutput;
|
|
646
983
|
if (qGateTensor) {
|
|
@@ -695,7 +1032,16 @@ export async function recordLayerAttentionGPU(
|
|
|
695
1032
|
hasLoRA: Boolean(loraO),
|
|
696
1033
|
oProjIsF16: oProjDtype === 'f16',
|
|
697
1034
|
});
|
|
698
|
-
|
|
1035
|
+
const canUseWideTileResidual = canUseQ4KWideTileResidualFusion({
|
|
1036
|
+
phase,
|
|
1037
|
+
session: mergedSessionRec,
|
|
1038
|
+
hasResidual: Boolean(residualTensor),
|
|
1039
|
+
residualMatches: Boolean(residualTensor && residualTensor.dtype === oProjInput.dtype),
|
|
1040
|
+
inputDtype: oProjInput.dtype,
|
|
1041
|
+
outputDtype: oProjOutputDtype,
|
|
1042
|
+
oProjDtype,
|
|
1043
|
+
hasLoRA: Boolean(loraO),
|
|
1044
|
+
});
|
|
699
1045
|
if (canUseFused && residualTensor) {
|
|
700
1046
|
// FUSED PATH: o_proj matmul + residual add in one dispatch
|
|
701
1047
|
output = await recordMatmulResidualFused(recorder, oProjInput, oProjBuf, residualTensor, {
|
|
@@ -704,7 +1050,7 @@ export async function recordLayerAttentionGPU(
|
|
|
704
1050
|
});
|
|
705
1051
|
residualFused = true;
|
|
706
1052
|
} else {
|
|
707
|
-
// STANDARD PATH: o_proj matmul only
|
|
1053
|
+
// STANDARD PATH: o_proj matmul only unless Q4K WideTile fuses residual in the epilogue
|
|
708
1054
|
output = await recordMatmul(recorder, oProjInput, oProjBuf, numTokens, hiddenSize, numHeads * headDim, {
|
|
709
1055
|
transposeB: 'auto',
|
|
710
1056
|
role: 'o_proj',
|
|
@@ -712,7 +1058,9 @@ export async function recordLayerAttentionGPU(
|
|
|
712
1058
|
kernelPath,
|
|
713
1059
|
outputDtype: oProjOutputDtype,
|
|
714
1060
|
executionPolicies: state.executionPolicies ?? null,
|
|
1061
|
+
residualTensor: canUseWideTileResidual ? residualTensor : null,
|
|
715
1062
|
});
|
|
1063
|
+
residualFused = canUseWideTileResidual;
|
|
716
1064
|
}
|
|
717
1065
|
// Release temporary buffer if we created it (original was not already on GPU)
|
|
718
1066
|
if (!isGpuBufferInstance(layerWeights.oProj) && !isWeightBuffer(layerWeights.oProj)) {
|
|
@@ -743,6 +1091,15 @@ export async function recordLayerAttentionGPU(
|
|
|
743
1091
|
}
|
|
744
1092
|
|
|
745
1093
|
finalOutput = output;
|
|
1094
|
+
if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
|
|
1095
|
+
enqueueRecordedTensorHealth(
|
|
1096
|
+
recorder,
|
|
1097
|
+
`L${layerIdx}.o_proj_HEALTH`,
|
|
1098
|
+
output,
|
|
1099
|
+
output.dtype,
|
|
1100
|
+
numTokens * hiddenSize
|
|
1101
|
+
);
|
|
1102
|
+
}
|
|
746
1103
|
|
|
747
1104
|
const buffersToTrack = [];
|
|
748
1105
|
if (output.buffer !== attnForProjection.buffer) {
|
|
@@ -764,13 +1121,17 @@ export async function recordLayerAttentionGPU(
|
|
|
764
1121
|
// These buffers are used by recorded operations that haven't executed yet.
|
|
765
1122
|
// Releasing them back to the pool would allow reuse before the encoder is submitted,
|
|
766
1123
|
// causing data corruption (especially for small decode buffers).
|
|
767
|
-
|
|
1124
|
+
if (qTensor) {
|
|
1125
|
+
recorder.trackTemporaryBuffer(qTensor.buffer);
|
|
1126
|
+
}
|
|
768
1127
|
if (qGateTensor) {
|
|
769
1128
|
recorder.trackTemporaryBuffer(qGateTensor.buffer);
|
|
770
1129
|
}
|
|
771
1130
|
if (!retainSharedKvBuffers) {
|
|
772
|
-
|
|
773
|
-
|
|
1131
|
+
if (kTensor) {
|
|
1132
|
+
recorder.trackTemporaryBuffer(kTensor.buffer);
|
|
1133
|
+
}
|
|
1134
|
+
if (vTensor?.buffer && vTensor.buffer !== kTensor?.buffer) {
|
|
774
1135
|
recorder.trackTemporaryBuffer(vTensor.buffer);
|
|
775
1136
|
}
|
|
776
1137
|
}
|
|
@@ -819,6 +1180,18 @@ export async function recordLayerAttentionGPU(
|
|
|
819
1180
|
if (attentionInputTemp && attentionInput?.buffer) {
|
|
820
1181
|
trackOnce(attentionInput.buffer);
|
|
821
1182
|
}
|
|
1183
|
+
if (decoderKTemp?.buffer) {
|
|
1184
|
+
trackOnce(decoderKTemp.buffer);
|
|
1185
|
+
}
|
|
1186
|
+
if (decoderVTemp?.buffer) {
|
|
1187
|
+
trackOnce(decoderVTemp.buffer);
|
|
1188
|
+
}
|
|
1189
|
+
if (decoderKVState?.ownedBuffers) {
|
|
1190
|
+
for (const buffer of decoderKVState.ownedBuffers) {
|
|
1191
|
+
trackOnce(buffer);
|
|
1192
|
+
}
|
|
1193
|
+
decoderKVState.ownedBuffers = null;
|
|
1194
|
+
}
|
|
822
1195
|
throw error;
|
|
823
1196
|
}
|
|
824
1197
|
}
|