doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
3
|
import { isGpuBufferInstance, isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
|
|
4
|
-
import { getDevice } from '../../../../gpu/device.js';
|
|
4
|
+
import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
|
|
5
5
|
import { getRuntimeConfig } from '../../../../config/runtime.js';
|
|
6
6
|
import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
|
|
7
7
|
import {
|
|
8
8
|
runMatmul,
|
|
9
9
|
runRMSNorm,
|
|
10
10
|
runRoPE,
|
|
11
|
+
canUseRoPEQK,
|
|
12
|
+
runRoPEQK,
|
|
11
13
|
runAttention,
|
|
12
14
|
runAttentionBDPA,
|
|
13
15
|
runAttentionTieredQuant,
|
|
@@ -34,6 +36,9 @@ import {
|
|
|
34
36
|
projectAttentionQKV,
|
|
35
37
|
applyAttentionQKNorm,
|
|
36
38
|
applyAttentionValueNorm,
|
|
39
|
+
hasAttentionProjectionDiagnostics,
|
|
40
|
+
hasAttentionStageDiagnostics,
|
|
41
|
+
resolveAttentionQKNormState,
|
|
37
42
|
} from './projections.js';
|
|
38
43
|
import { prepareAttentionProjectionInput } from './output-projection.js';
|
|
39
44
|
|
|
@@ -47,6 +52,7 @@ import {
|
|
|
47
52
|
} from '../../../../config/kernel-path-loader.js';
|
|
48
53
|
import {
|
|
49
54
|
resolveKVCacheState,
|
|
55
|
+
createDiffusionGemmaDecoderKVState,
|
|
50
56
|
buildAttentionDispatchParams,
|
|
51
57
|
buildAttentionInputsData,
|
|
52
58
|
} from './dispatch-params.js';
|
|
@@ -63,6 +69,26 @@ import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js'
|
|
|
63
69
|
|
|
64
70
|
const ATTENTION_DTYPE_LOGGED = new Set();
|
|
65
71
|
|
|
72
|
+
function isWideTileQ4KPhaseEnabled(session, phase) {
|
|
73
|
+
return phase === 'decode'
|
|
74
|
+
? session?.useWideTileQ4KDecode === true
|
|
75
|
+
: session?.useWideTileQ4KPrefill === true;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function canUseQ4KWideTileResidualFusion(options = {}) {
|
|
79
|
+
return options.hasResidual === true
|
|
80
|
+
&& options.oProjDtype === 'q4k'
|
|
81
|
+
&& options.inputDtype === 'f32'
|
|
82
|
+
&& options.outputDtype === 'f32'
|
|
83
|
+
&& options.residualMatches === true
|
|
84
|
+
&& options.hasLoRA !== true
|
|
85
|
+
&& getKernelCapabilities().hasF16 === true
|
|
86
|
+
&& getKernelCapabilities().hasSubgroups === true
|
|
87
|
+
&& options.session?.useWideTileResidualFusion === true
|
|
88
|
+
&& options.session?.retainQ4KMaterialization === true
|
|
89
|
+
&& isWideTileQ4KPhaseEnabled(options.session, options.phase);
|
|
90
|
+
}
|
|
91
|
+
|
|
66
92
|
function assertAttentionDtypeTransitionAllowed(state, fromDtype, toDtype, detail, transitionDeclaredBy = null) {
|
|
67
93
|
assertImplicitDtypeTransitionAllowed({
|
|
68
94
|
executionPolicies: state?.executionPolicies ?? null,
|
|
@@ -108,6 +134,8 @@ export async function runLayerAttentionGPU(
|
|
|
108
134
|
multimodalBidirectionalSpan = null,
|
|
109
135
|
sharedKVSourceLayerIdx = null,
|
|
110
136
|
storeSharedKV = false,
|
|
137
|
+
diffusionGemmaDecoder = false,
|
|
138
|
+
precomputedInputNorm = null,
|
|
111
139
|
} = config;
|
|
112
140
|
|
|
113
141
|
const device = getDevice();
|
|
@@ -130,6 +158,9 @@ export async function runLayerAttentionGPU(
|
|
|
130
158
|
const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: useF16Activations });
|
|
131
159
|
const kvCacheDtype = attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? kvCacheFallback;
|
|
132
160
|
const allowF16Attention = useF16Activations && kvCacheDtype === 'f16';
|
|
161
|
+
const attentionInputTransitionDeclaredBy = attentionPrecisionContract.explicitInputDtype
|
|
162
|
+
? 'step_precision'
|
|
163
|
+
: null;
|
|
133
164
|
let attentionInput = input;
|
|
134
165
|
let attentionInputTemp = false;
|
|
135
166
|
let normed = attentionInput;
|
|
@@ -143,12 +174,16 @@ export async function runLayerAttentionGPU(
|
|
|
143
174
|
let finalOutput = null;
|
|
144
175
|
let oProjInputTemp = null;
|
|
145
176
|
let valueAliasesKey = false;
|
|
177
|
+
let decoderKVState = null;
|
|
178
|
+
let decoderKTemp = null;
|
|
179
|
+
let decoderVTemp = null;
|
|
146
180
|
if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
|
|
147
181
|
assertAttentionDtypeTransitionAllowed(
|
|
148
182
|
state,
|
|
149
183
|
input.dtype,
|
|
150
184
|
attentionActivationDtype,
|
|
151
|
-
'The attention kernel selection would widen the input implicitly.'
|
|
185
|
+
'The attention kernel selection would widen the input implicitly.',
|
|
186
|
+
attentionInputTransitionDeclaredBy
|
|
152
187
|
);
|
|
153
188
|
attentionInput = attentionActivationDtype === 'f16'
|
|
154
189
|
? await castF32ToF16(input)
|
|
@@ -216,7 +251,28 @@ export async function runLayerAttentionGPU(
|
|
|
216
251
|
&& attentionInput.dtype === 'f32';
|
|
217
252
|
|
|
218
253
|
try {
|
|
219
|
-
if (
|
|
254
|
+
if (precomputedInputNorm) {
|
|
255
|
+
if (
|
|
256
|
+
isPrefill
|
|
257
|
+
|| numTokens !== 1
|
|
258
|
+
|| skipInputNorm
|
|
259
|
+
|| !layerWeights.inputNorm
|
|
260
|
+
|| !getNormWeightBuffer
|
|
261
|
+
|| precomputedInputNorm.dtype !== attentionInput.dtype
|
|
262
|
+
|| precomputedInputNorm.dtype !== 'f32'
|
|
263
|
+
) {
|
|
264
|
+
throw new Error(`Layer ${layerIdx} received an incompatible precomputed input norm tensor.`);
|
|
265
|
+
}
|
|
266
|
+
normed = precomputedInputNorm;
|
|
267
|
+
await runProbes('post_input_norm', normed.buffer, {
|
|
268
|
+
layerIdx,
|
|
269
|
+
numTokens,
|
|
270
|
+
hiddenSize,
|
|
271
|
+
probes: state.debugProbes,
|
|
272
|
+
operatorDiagnostics: state.operatorDiagnostics,
|
|
273
|
+
dtype: normed.dtype,
|
|
274
|
+
});
|
|
275
|
+
} else if (canFuseInputNormIntoProjections) {
|
|
220
276
|
const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
221
277
|
fusedNormWeight = normWeightBuf;
|
|
222
278
|
fusedNormEps = rmsNormEps;
|
|
@@ -305,7 +361,24 @@ export async function runLayerAttentionGPU(
|
|
|
305
361
|
}),
|
|
306
362
|
});
|
|
307
363
|
let usedFusedQKV = false;
|
|
308
|
-
|
|
364
|
+
let qkNormApplied = false;
|
|
365
|
+
let ropeApplied = false;
|
|
366
|
+
const qkNormState = resolveAttentionQKNormState({ config, layerWeights, layerIdx, reusesSharedKV });
|
|
367
|
+
const qkNormProjectionDiagnostics = hasAttentionProjectionDiagnostics(state)
|
|
368
|
+
|| kernelTrace.enabled
|
|
369
|
+
|| isKernelDebugEnabled(layerIdx)
|
|
370
|
+
|| (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !!debugCheckBuffer);
|
|
371
|
+
const qkNormRoPEDiagnostics = hasAttentionStageDiagnostics(
|
|
372
|
+
state,
|
|
373
|
+
['q_proj', 'k_proj', 'v_proj', 'q_norm', 'k_norm']
|
|
374
|
+
)
|
|
375
|
+
|| kernelTrace.enabled
|
|
376
|
+
|| isKernelDebugEnabled(layerIdx)
|
|
377
|
+
|| (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !!debugCheckBuffer);
|
|
378
|
+
const runtimeSession = getRuntimeConfig()?.inference?.session;
|
|
379
|
+
const qkNormFusionFlag = runtimeSession?.useFusedQKVSplitQKNorm === true;
|
|
380
|
+
const qkNormRoPEFusionFlag = runtimeSession?.useFusedQKVSplitQKNormRoPE === true;
|
|
381
|
+
({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey, qkNormApplied, ropeApplied } = await projectAttentionQKV({
|
|
309
382
|
recorder: null,
|
|
310
383
|
normed,
|
|
311
384
|
layerWeights,
|
|
@@ -333,6 +406,35 @@ export async function runLayerAttentionGPU(
|
|
|
333
406
|
fusedNormWeight,
|
|
334
407
|
fusedNormEps,
|
|
335
408
|
fusedNormOffset,
|
|
409
|
+
qkNormFusion: {
|
|
410
|
+
enabled: qkNormFusionFlag && qkNormState.wantsQKNorm,
|
|
411
|
+
getNormWeightBuffer,
|
|
412
|
+
rmsNormEps,
|
|
413
|
+
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
414
|
+
skipKNorm: qkNormState.skipKNorm,
|
|
415
|
+
allowUnitQKNorm: qkNormState.allowUnitQKNorm,
|
|
416
|
+
projectionDiagnosticsEnabled: qkNormProjectionDiagnostics,
|
|
417
|
+
},
|
|
418
|
+
qkNormRoPEFusion: {
|
|
419
|
+
enabled: qkNormRoPEFusionFlag
|
|
420
|
+
&& qkNormState.wantsQKNorm
|
|
421
|
+
&& !disableRoPE
|
|
422
|
+
&& !!state.ropeFreqsCos
|
|
423
|
+
&& !!state.ropeFreqsSin,
|
|
424
|
+
getNormWeightBuffer,
|
|
425
|
+
rmsNormEps,
|
|
426
|
+
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
427
|
+
skipKNorm: qkNormState.skipKNorm,
|
|
428
|
+
allowUnitQKNorm: qkNormState.allowUnitQKNorm,
|
|
429
|
+
projectionDiagnosticsEnabled: qkNormRoPEDiagnostics,
|
|
430
|
+
freqsCos: state.ropeFreqsCos,
|
|
431
|
+
freqsSin: state.ropeFreqsSin,
|
|
432
|
+
startPos: currentSeqLen,
|
|
433
|
+
rotaryDim: config.ropeRotaryDim,
|
|
434
|
+
pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
|
|
435
|
+
interleaved: config.ropeInterleaved,
|
|
436
|
+
reusesSharedKV,
|
|
437
|
+
},
|
|
336
438
|
}));
|
|
337
439
|
// Release the in-flight norm weight buffer used by the fused projections.
|
|
338
440
|
if (fusedNormWeight && fusedNormWeightOwned) {
|
|
@@ -406,17 +508,19 @@ export async function runLayerAttentionGPU(
|
|
|
406
508
|
}
|
|
407
509
|
|
|
408
510
|
// Optional per-head Q/K normalization
|
|
409
|
-
const wantsQKNorm = config.queryKeyNorm === true;
|
|
410
|
-
const hasQNorm = !!layerWeights.qNorm;
|
|
411
|
-
const hasKNorm = !!layerWeights.kNorm;
|
|
412
511
|
if (isKernelDebugEnabled(layerIdx)) {
|
|
413
|
-
logKernelStep(
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
512
|
+
logKernelStep(
|
|
513
|
+
'qk_norm',
|
|
514
|
+
{
|
|
515
|
+
layerIdx,
|
|
516
|
+
label: `hasQ=${qkNormState.hasQNorm} hasK=${qkNormState.hasKNorm} ` +
|
|
517
|
+
`wants=${qkNormState.wantsQKNorm} unit=${qkNormState.allowUnitQKNorm}`,
|
|
518
|
+
}
|
|
519
|
+
);
|
|
417
520
|
}
|
|
418
521
|
|
|
419
|
-
(
|
|
522
|
+
if (qkNormState.wantsQKNorm && !qkNormApplied) {
|
|
523
|
+
({ qTensor, kTensor } = await applyAttentionQKNorm({
|
|
420
524
|
recorder: null,
|
|
421
525
|
qTensor,
|
|
422
526
|
kTensor,
|
|
@@ -429,7 +533,7 @@ export async function runLayerAttentionGPU(
|
|
|
429
533
|
headDim,
|
|
430
534
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
431
535
|
releaseTemporary: (buffer) => releaseBuffer(buffer),
|
|
432
|
-
skipKNorm:
|
|
536
|
+
skipKNorm: qkNormState.skipKNorm,
|
|
433
537
|
onQNormApplied: isKernelDebugEnabled(layerIdx)
|
|
434
538
|
? async (tensor) => {
|
|
435
539
|
await dumpTokenVector(tensor.buffer, 'Q_norm', {
|
|
@@ -449,9 +553,11 @@ export async function runLayerAttentionGPU(
|
|
|
449
553
|
dtype: tensor.dtype,
|
|
450
554
|
});
|
|
451
555
|
}
|
|
452
|
-
|
|
556
|
+
: null,
|
|
453
557
|
retainKInput: valueAliasesKey,
|
|
454
|
-
|
|
558
|
+
allowUnitQKNorm: qkNormState.allowUnitQKNorm,
|
|
559
|
+
}));
|
|
560
|
+
}
|
|
455
561
|
|
|
456
562
|
if (config.valueNorm === true && !reusesSharedKV) {
|
|
457
563
|
const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
|
|
@@ -478,23 +584,25 @@ export async function runLayerAttentionGPU(
|
|
|
478
584
|
});
|
|
479
585
|
}
|
|
480
586
|
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
numTokens,
|
|
484
|
-
hiddenSize: numHeads * headDim,
|
|
485
|
-
probes: state.debugProbes,
|
|
486
|
-
operatorDiagnostics: state.operatorDiagnostics,
|
|
487
|
-
dtype: qTensor.dtype,
|
|
488
|
-
});
|
|
489
|
-
if (!reusesSharedKV) {
|
|
490
|
-
await runProbes('k_norm', kTensor.buffer, {
|
|
587
|
+
if (qkNormState.wantsQKNorm) {
|
|
588
|
+
await runProbes('q_norm', qTensor.buffer, {
|
|
491
589
|
layerIdx,
|
|
492
590
|
numTokens,
|
|
493
|
-
hiddenSize:
|
|
591
|
+
hiddenSize: numHeads * headDim,
|
|
494
592
|
probes: state.debugProbes,
|
|
495
593
|
operatorDiagnostics: state.operatorDiagnostics,
|
|
496
|
-
dtype:
|
|
594
|
+
dtype: qTensor.dtype,
|
|
497
595
|
});
|
|
596
|
+
if (!reusesSharedKV) {
|
|
597
|
+
await runProbes('k_norm', kTensor.buffer, {
|
|
598
|
+
layerIdx,
|
|
599
|
+
numTokens,
|
|
600
|
+
hiddenSize: numKVHeads * headDim,
|
|
601
|
+
probes: state.debugProbes,
|
|
602
|
+
operatorDiagnostics: state.operatorDiagnostics,
|
|
603
|
+
dtype: kTensor.dtype,
|
|
604
|
+
});
|
|
605
|
+
}
|
|
498
606
|
}
|
|
499
607
|
|
|
500
608
|
if (normed !== attentionInput) releaseBuffer(normed.buffer);
|
|
@@ -502,26 +610,32 @@ export async function runLayerAttentionGPU(
|
|
|
502
610
|
|
|
503
611
|
// 3. RoPE (modifies tensor in-place)
|
|
504
612
|
|
|
505
|
-
if (!disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
|
|
506
|
-
|
|
507
|
-
numHeads,
|
|
613
|
+
if (!ropeApplied && !disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
|
|
614
|
+
const ropeOptions = {
|
|
508
615
|
headDim,
|
|
509
616
|
rotaryDim: config.ropeRotaryDim,
|
|
510
617
|
pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
|
|
511
618
|
interleaved: config.ropeInterleaved,
|
|
512
619
|
startPos: currentSeqLen,
|
|
513
620
|
executionPolicies: state.executionPolicies ?? null,
|
|
514
|
-
}
|
|
515
|
-
if (
|
|
516
|
-
await
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
|
|
521
|
-
interleaved: config.ropeInterleaved,
|
|
522
|
-
startPos: currentSeqLen,
|
|
523
|
-
executionPolicies: state.executionPolicies ?? null,
|
|
621
|
+
};
|
|
622
|
+
if (canUseRoPEQK(qTensor, kTensor, { reusesSharedKV })) {
|
|
623
|
+
await runRoPEQK(qTensor, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
|
|
624
|
+
...ropeOptions,
|
|
625
|
+
numQHeads: numHeads,
|
|
626
|
+
numKVHeads,
|
|
524
627
|
});
|
|
628
|
+
} else {
|
|
629
|
+
await runRoPE(qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
|
|
630
|
+
...ropeOptions,
|
|
631
|
+
numHeads,
|
|
632
|
+
});
|
|
633
|
+
if (!reusesSharedKV) {
|
|
634
|
+
await runRoPE(kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
|
|
635
|
+
...ropeOptions,
|
|
636
|
+
numHeads: numKVHeads,
|
|
637
|
+
});
|
|
638
|
+
}
|
|
525
639
|
}
|
|
526
640
|
|
|
527
641
|
// Trace RoPE outputs
|
|
@@ -582,7 +696,7 @@ export async function runLayerAttentionGPU(
|
|
|
582
696
|
|
|
583
697
|
// 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
|
|
584
698
|
|
|
585
|
-
if (state.kvCache?.hasGPUCache?.()) {
|
|
699
|
+
if (!diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
|
|
586
700
|
if (state.kvCache.kvDtype === 'f16') {
|
|
587
701
|
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
588
702
|
if (kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
|
|
@@ -610,11 +724,66 @@ export async function runLayerAttentionGPU(
|
|
|
610
724
|
}
|
|
611
725
|
|
|
612
726
|
// Resolve KV cache state and build dispatch parameters (shared with record.js)
|
|
613
|
-
|
|
727
|
+
let kvState;
|
|
728
|
+
if (diffusionGemmaDecoder) {
|
|
729
|
+
const decoderKVDtype = kvCacheDtype ?? kTensor.dtype;
|
|
730
|
+
let decoderKTensor = kTensor;
|
|
731
|
+
let decoderVTensor = vTensor;
|
|
732
|
+
if (decoderKVDtype === 'f16' && decoderKTensor.dtype !== 'f16') {
|
|
733
|
+
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
734
|
+
if (!hasExplicitF16KvContract) {
|
|
735
|
+
assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f16', 'DiffusionGemma decoder K would be narrowed implicitly.');
|
|
736
|
+
}
|
|
737
|
+
decoderKTensor = await castF32ToF16(decoderKTensor);
|
|
738
|
+
decoderKTemp = decoderKTensor;
|
|
739
|
+
} else if (decoderKVDtype === 'f32' && decoderKTensor.dtype !== 'f32') {
|
|
740
|
+
assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f32', 'DiffusionGemma decoder K would be widened implicitly.');
|
|
741
|
+
decoderKTensor = await castF16ToF32(decoderKTensor);
|
|
742
|
+
decoderKTemp = decoderKTensor;
|
|
743
|
+
}
|
|
744
|
+
if (decoderKVDtype === 'f16' && decoderVTensor.dtype !== 'f16') {
|
|
745
|
+
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
746
|
+
if (!hasExplicitF16KvContract) {
|
|
747
|
+
assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f16', 'DiffusionGemma decoder V would be narrowed implicitly.');
|
|
748
|
+
}
|
|
749
|
+
decoderVTensor = await castF32ToF16(decoderVTensor);
|
|
750
|
+
decoderVTemp = decoderVTensor;
|
|
751
|
+
} else if (decoderKVDtype === 'f32' && decoderVTensor.dtype !== 'f32') {
|
|
752
|
+
assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f32', 'DiffusionGemma decoder V would be widened implicitly.');
|
|
753
|
+
decoderVTensor = await castF16ToF32(decoderVTensor);
|
|
754
|
+
decoderVTemp = decoderVTensor;
|
|
755
|
+
}
|
|
756
|
+
kvState = await createDiffusionGemmaDecoderKVState({
|
|
757
|
+
state,
|
|
758
|
+
layerIdx,
|
|
759
|
+
kTensor: decoderKTensor,
|
|
760
|
+
vTensor: decoderVTensor,
|
|
761
|
+
currentSeqLen,
|
|
762
|
+
numTokens,
|
|
763
|
+
numKVHeads,
|
|
764
|
+
headDim,
|
|
765
|
+
layerType,
|
|
766
|
+
slidingWindow,
|
|
767
|
+
kvDtype: decoderKVDtype,
|
|
768
|
+
});
|
|
769
|
+
if (decoderKTemp) {
|
|
770
|
+
releaseBuffer(decoderKTemp.buffer);
|
|
771
|
+
decoderKTemp = null;
|
|
772
|
+
}
|
|
773
|
+
if (decoderVTemp && decoderVTemp.buffer !== decoderKTensor.buffer) {
|
|
774
|
+
releaseBuffer(decoderVTemp.buffer);
|
|
775
|
+
}
|
|
776
|
+
decoderVTemp = null;
|
|
777
|
+
} else {
|
|
778
|
+
kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens);
|
|
779
|
+
}
|
|
614
780
|
const dispatchConfig = {
|
|
615
781
|
layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
|
|
616
|
-
slidingWindow
|
|
617
|
-
|
|
782
|
+
slidingWindow: diffusionGemmaDecoder ? null : slidingWindow,
|
|
783
|
+
layerType,
|
|
784
|
+
layerTypes: config.layerTypes,
|
|
785
|
+
queryPreAttnScalar,
|
|
786
|
+
causalAttention: diffusionGemmaDecoder ? false : config.causalAttention,
|
|
618
787
|
activationDtype: attentionActivationDtype,
|
|
619
788
|
kvCacheDtype: attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? null,
|
|
620
789
|
};
|
|
@@ -823,8 +992,8 @@ export async function runLayerAttentionGPU(
|
|
|
823
992
|
// see comment above at the rmsNorm fusion check). The kernel itself
|
|
824
993
|
// enforces remaining preconditions (head_dim=256, etc.).
|
|
825
994
|
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
826
|
-
const useFlashPrefill = mergedSession?.useFlashPrefillAttention === true && numTokens > 1;
|
|
827
|
-
const useOrtFlashPrefill = mergedSession?.useOrtFlashPrefillAttention === true && numTokens > 1;
|
|
995
|
+
const useFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useFlashPrefillAttention === true && numTokens > 1;
|
|
996
|
+
const useOrtFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useOrtFlashPrefillAttention === true && numTokens > 1;
|
|
828
997
|
const result = await runAttention(qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
|
|
829
998
|
seqLen: numTokens,
|
|
830
999
|
kvLen: kvState.kvLenForAttention,
|
|
@@ -858,7 +1027,16 @@ export async function runLayerAttentionGPU(
|
|
|
858
1027
|
throw new Error(`Unsupported attention kernel variant "${attentionKernelVariant}" at layer ${layerIdx}`);
|
|
859
1028
|
}
|
|
860
1029
|
|
|
861
|
-
|
|
1030
|
+
try {
|
|
1031
|
+
attnOutput = await runAttentionKernel();
|
|
1032
|
+
} finally {
|
|
1033
|
+
if (decoderKVState?.ownedBuffers) {
|
|
1034
|
+
for (const buffer of decoderKVState.ownedBuffers) {
|
|
1035
|
+
releaseBuffer(buffer);
|
|
1036
|
+
}
|
|
1037
|
+
decoderKVState.ownedBuffers = null;
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
862
1040
|
await runProbes('attn_core_out', attnOutput.buffer, {
|
|
863
1041
|
layerIdx,
|
|
864
1042
|
numTokens,
|
|
@@ -947,6 +1125,7 @@ export async function runLayerAttentionGPU(
|
|
|
947
1125
|
// Use fused o_proj + residual for decode when possible
|
|
948
1126
|
// Note: dtype from WeightBuffer metadata (buffer-dtypes WeakMap removed)
|
|
949
1127
|
const oProjDtype = getWeightDtype(oProjBuf);
|
|
1128
|
+
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
950
1129
|
const canUseFused = selectRuleValue('inference', 'attention', 'useFusedOProjResidual', {
|
|
951
1130
|
allowFusedResidual: shouldUseFusedMatmulResidual(numTokens),
|
|
952
1131
|
hasResidual: Boolean(residualTensor),
|
|
@@ -956,7 +1135,16 @@ export async function runLayerAttentionGPU(
|
|
|
956
1135
|
hasLoRA: Boolean(loraO),
|
|
957
1136
|
oProjIsF16: oProjDtype === 'f16',
|
|
958
1137
|
}); // GEMV kernel expects f16 weights
|
|
959
|
-
|
|
1138
|
+
const canUseWideTileResidual = canUseQ4KWideTileResidualFusion({
|
|
1139
|
+
phase,
|
|
1140
|
+
session: mergedSession,
|
|
1141
|
+
hasResidual: Boolean(residualTensor),
|
|
1142
|
+
residualMatches: Boolean(residualTensor && residualTensor.dtype === oProjInput.dtype),
|
|
1143
|
+
inputDtype: oProjInput.dtype,
|
|
1144
|
+
outputDtype: oProjOutputDtype,
|
|
1145
|
+
oProjDtype,
|
|
1146
|
+
hasLoRA: Boolean(loraO),
|
|
1147
|
+
});
|
|
960
1148
|
if (canUseFused && residualTensor) {
|
|
961
1149
|
// FUSED PATH: o_proj matmul + residual add in one dispatch
|
|
962
1150
|
output = await runMatmulResidualFused(oProjInput, oProjBuf, residualTensor, {
|
|
@@ -969,7 +1157,7 @@ export async function runLayerAttentionGPU(
|
|
|
969
1157
|
trace.attn(layerIdx, `Using fused o_proj+residual path`);
|
|
970
1158
|
}
|
|
971
1159
|
} else {
|
|
972
|
-
// STANDARD PATH: o_proj matmul only
|
|
1160
|
+
// STANDARD PATH: o_proj matmul only unless Q4K WideTile fuses residual in the epilogue
|
|
973
1161
|
output = await runMatmul(oProjInput, oProjBuf, numTokens, hiddenSize, numHeads * headDim, {
|
|
974
1162
|
transposeB: 'auto',
|
|
975
1163
|
role: 'o_proj',
|
|
@@ -977,7 +1165,9 @@ export async function runLayerAttentionGPU(
|
|
|
977
1165
|
kernelPath,
|
|
978
1166
|
outputDtype: oProjOutputDtype,
|
|
979
1167
|
executionPolicies: state.executionPolicies ?? null,
|
|
1168
|
+
residualTensor: canUseWideTileResidual ? residualTensor : null,
|
|
980
1169
|
});
|
|
1170
|
+
residualFused = canUseWideTileResidual;
|
|
981
1171
|
}
|
|
982
1172
|
// Release temporary buffer if we created it (original was not already on GPU)
|
|
983
1173
|
if (!isGpuBufferInstance(layerWeights.oProj) && !isWeightBuffer(layerWeights.oProj)) {
|
|
@@ -1111,6 +1301,18 @@ export async function runLayerAttentionGPU(
|
|
|
1111
1301
|
if (attentionInputTemp && attentionInput?.buffer) {
|
|
1112
1302
|
releaseOnce(attentionInput.buffer);
|
|
1113
1303
|
}
|
|
1304
|
+
if (decoderKTemp?.buffer) {
|
|
1305
|
+
releaseOnce(decoderKTemp.buffer);
|
|
1306
|
+
}
|
|
1307
|
+
if (decoderVTemp?.buffer) {
|
|
1308
|
+
releaseOnce(decoderVTemp.buffer);
|
|
1309
|
+
}
|
|
1310
|
+
if (decoderKVState?.ownedBuffers) {
|
|
1311
|
+
for (const buffer of decoderKVState.ownedBuffers) {
|
|
1312
|
+
releaseOnce(buffer);
|
|
1313
|
+
}
|
|
1314
|
+
decoderKVState.ownedBuffers = null;
|
|
1315
|
+
}
|
|
1114
1316
|
throw error;
|
|
1115
1317
|
}
|
|
1116
1318
|
}
|
|
@@ -39,8 +39,12 @@ export interface AttentionConfig {
|
|
|
39
39
|
attnSoftcap?: number;
|
|
40
40
|
/** Gemma 2 attention scaling: uses head_dim (256) instead of sqrt(head_dim) (16). */
|
|
41
41
|
queryPreAttnScalar?: number;
|
|
42
|
-
/** Apply query/key RMSNorm
|
|
42
|
+
/** Apply query/key RMSNorm where per-head weights are declared. */
|
|
43
43
|
queryKeyNorm?: boolean;
|
|
44
|
+
/** Layers that apply query/key RMSNorm; null means all layers when queryKeyNorm=true. */
|
|
45
|
+
queryKeyNormLayers?: number[] | null;
|
|
46
|
+
/** Layers that carry explicit Q/K RMSNorm scale tensors; null means missing weights are invalid. */
|
|
47
|
+
queryKeyNormWeightLayers?: number[] | null;
|
|
44
48
|
/** Apply unit-scale value RMSNorm before attention. */
|
|
45
49
|
valueNorm?: boolean;
|
|
46
50
|
/** Apply sigmoid gate from q_proj split to attention output before o_proj. */
|
|
@@ -109,9 +109,23 @@ export interface TensorInfo {
|
|
|
109
109
|
dtype?: string;
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
+
export interface ParsedSessionSettings {
|
|
113
|
+
prefillChunkSubmitMode: 'sync' | 'async' | null;
|
|
114
|
+
prefillTokenChunkSize: number | null;
|
|
115
|
+
useFlashPrefillAttention: boolean | null;
|
|
116
|
+
useWideTileQ4KPrefill: boolean | null;
|
|
117
|
+
useWideTileQ4KDecode: boolean | null;
|
|
118
|
+
useSandwichRMSNormPairFusion: boolean | null;
|
|
119
|
+
usePostFfnNextInputRMSNormPairFusion: boolean | null;
|
|
120
|
+
useFusedQKVSplitQKNorm: boolean | null;
|
|
121
|
+
useFusedQKVSplitQKNormRoPE: boolean | null;
|
|
122
|
+
retainQ4KMaterialization: boolean | null;
|
|
123
|
+
}
|
|
124
|
+
|
|
112
125
|
export type Manifest = RuntimeModelContract;
|
|
113
126
|
|
|
114
127
|
export interface ParsedModelConfig {
|
|
128
|
+
modelType: string;
|
|
115
129
|
numLayers: number;
|
|
116
130
|
hiddenSize: number;
|
|
117
131
|
intermediateSize: number;
|
|
@@ -130,7 +144,8 @@ export interface ParsedModelConfig {
|
|
|
130
144
|
useMoE: boolean;
|
|
131
145
|
numExperts: number;
|
|
132
146
|
moeTopK: number;
|
|
133
|
-
expertFormat: 'mixtral' | 'gpt-oss' | null;
|
|
147
|
+
expertFormat: 'mixtral' | 'gpt-oss' | 'gemma4' | null;
|
|
148
|
+
moeExpertIntermediateSize: number;
|
|
134
149
|
slidingWindow: number | null;
|
|
135
150
|
ropeTheta: number;
|
|
136
151
|
ropeLocalTheta: number | null;
|
|
@@ -162,6 +177,7 @@ export interface ParsedModelConfig {
|
|
|
162
177
|
embeddingVocabSize: number | null;
|
|
163
178
|
embeddingPostprocessor: ManifestEmbeddingPostprocessorSchema | null;
|
|
164
179
|
hiddenActivation: ActivationType;
|
|
180
|
+
ffnBranchMode: 'auto' | 'dense' | 'moe' | 'dense_plus_moe';
|
|
165
181
|
useDoubleWideMlp: boolean;
|
|
166
182
|
swigluLimit: number | null;
|
|
167
183
|
stopTokenIds: number[];
|
|
@@ -178,6 +194,8 @@ export interface ParsedModelConfig {
|
|
|
178
194
|
finalLogitSoftcapping: number | null;
|
|
179
195
|
attnLogitSoftcapping: number | null;
|
|
180
196
|
queryKeyNorm: boolean;
|
|
197
|
+
queryKeyNormLayers: number[] | null;
|
|
198
|
+
queryKeyNormWeightLayers: number[] | null;
|
|
181
199
|
valueNorm: boolean;
|
|
182
200
|
attentionOutputGate: boolean;
|
|
183
201
|
queryPreAttnScalar: number;
|
|
@@ -186,7 +204,9 @@ export interface ParsedModelConfig {
|
|
|
186
204
|
chatTemplateEnabled: boolean;
|
|
187
205
|
chatTemplateThinking: boolean | null;
|
|
188
206
|
decodeStrategy: 'incremental' | 'replay_prefill';
|
|
207
|
+
diffusionGemma: ManifestInferenceSchema['diffusionGemma'];
|
|
189
208
|
perLayerInputsSession: ExecutionV1PerLayerInputsSessionSchema;
|
|
209
|
+
sessionSettings: ParsedSessionSettings;
|
|
190
210
|
kernelPath?: KernelPathRef;
|
|
191
211
|
visionConfig?: VisionConfig | null;
|
|
192
212
|
audioConfig?: AudioEncoderConfig | null;
|
|
@@ -246,6 +266,7 @@ export interface AudioEncoderConfig {
|
|
|
246
266
|
|
|
247
267
|
export function getStopTokenIds(manifest: Manifest): number[];
|
|
248
268
|
export function resolveLayerIntermediateSize(config: ParsedModelConfig, layerIdx: number): number;
|
|
269
|
+
export function assertSupportedManifestInference(manifest: Manifest): void;
|
|
249
270
|
|
|
250
271
|
/**
|
|
251
272
|
* Extended manifest with inference config for manifest-first parsing.
|