doppler-gpu 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +67 -34
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.js +5 -1
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/version.js +1 -1
|
@@ -16,7 +16,11 @@ import type { ParsedModelConfig, Manifest } from './config.js';
|
|
|
16
16
|
import type { KernelCapabilities } from '../../../gpu/device.js';
|
|
17
17
|
import type { WeightBuffer, CpuWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
18
18
|
import type { LoadedEmbeddingPostprocessor } from '../../../loader/final-weights-loader.js';
|
|
19
|
-
import type {
|
|
19
|
+
import type {
|
|
20
|
+
DiffusionGemmaSelfConditioningWeights,
|
|
21
|
+
LoaderLoadTiming,
|
|
22
|
+
PerLayerInputWeights,
|
|
23
|
+
} from '../../../loader/loader-types.js';
|
|
20
24
|
import {
|
|
21
25
|
KVCache,
|
|
22
26
|
SlidingWindowKVCache,
|
|
@@ -57,6 +61,7 @@ export interface PipelineStorageContext {
|
|
|
57
61
|
) => AsyncIterable<ArrayBuffer | Uint8Array>;
|
|
58
62
|
loadTokenizerJson?: () => Promise<Record<string, unknown> | string | null | undefined>;
|
|
59
63
|
loadTokenizerModel?: (path?: string) => Promise<ArrayBuffer | Uint8Array | null | undefined>;
|
|
64
|
+
loadAuxiliaryFile?: (path: string) => Promise<ArrayBuffer | Uint8Array | null | undefined>;
|
|
60
65
|
loadTensorsJson?: () => Promise<string | Record<string, unknown> | null | undefined>;
|
|
61
66
|
verifyHashes?: boolean;
|
|
62
67
|
close?: () => Promise<void>;
|
|
@@ -219,8 +224,10 @@ export interface WeightLoadResult {
|
|
|
219
224
|
lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
|
|
220
225
|
finalNorm: GPUBuffer | Float32Array | null;
|
|
221
226
|
embeddingPostprocessor: LoadedEmbeddingPostprocessor | null;
|
|
227
|
+
diffusionGemmaSelfConditioning: DiffusionGemmaSelfConditioningWeights | null;
|
|
222
228
|
perLayerInputWeights: PerLayerInputWeights | null;
|
|
223
229
|
layerRouterWeights: Map<number, RouterWeights>;
|
|
230
|
+
loadTiming: LoaderLoadTiming | null;
|
|
224
231
|
}
|
|
225
232
|
|
|
226
233
|
export interface ResolvedQ4KConfig {
|
|
@@ -418,6 +418,20 @@ function usesMixedGeometryKVCache(modelConfig) {
|
|
|
418
418
|
return hasExplicitLayerTypes && (hasMixedAttentionGeometry || hasSharedKvLayers);
|
|
419
419
|
}
|
|
420
420
|
|
|
421
|
+
function resolveMixedGeometrySlidingLayerLayout(modelConfig) {
|
|
422
|
+
const diffusionGemmaContract = modelConfig?.diffusionGemma ?? null;
|
|
423
|
+
if (diffusionGemmaContract == null) {
|
|
424
|
+
return 'ring';
|
|
425
|
+
}
|
|
426
|
+
if (diffusionGemmaContract.decoderCacheMode === 'encoder_kv_readonly_canvas_concat') {
|
|
427
|
+
return 'contiguous';
|
|
428
|
+
}
|
|
429
|
+
throw new Error(
|
|
430
|
+
`Unsupported DiffusionGemma decoderCacheMode="${String(diffusionGemmaContract.decoderCacheMode)}" ` +
|
|
431
|
+
'for mixed-geometry KV cache.'
|
|
432
|
+
);
|
|
433
|
+
}
|
|
434
|
+
|
|
421
435
|
function getRequiredKernelMaxKVLen(operation, variant, label) {
|
|
422
436
|
const config = getKernelConfig(operation, variant);
|
|
423
437
|
const maxKVLen = config.variantMetadata?.maxKVLen;
|
|
@@ -693,6 +707,7 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
|
|
|
693
707
|
globalNumHeads: modelConfig.numGlobalKVHeads ?? modelConfig.numKVHeads,
|
|
694
708
|
globalHeadDim: modelConfig.globalHeadDim ?? null,
|
|
695
709
|
slidingWindow,
|
|
710
|
+
slidingLayerLayout: resolveMixedGeometrySlidingLayerLayout(modelConfig),
|
|
696
711
|
layerTypes: modelConfig.layerTypes,
|
|
697
712
|
});
|
|
698
713
|
} else if (modelConfig.slidingWindow && !forceContiguousKVCache && cacheLayout !== 'paged' && cacheLayout !== 'tiered' && cacheLayout !== 'bdpa') {
|
|
@@ -895,6 +910,9 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
895
910
|
verify: verifyHashes,
|
|
896
911
|
loadShardRange,
|
|
897
912
|
streamShardRange,
|
|
913
|
+
loadAuxiliaryFile: typeof runtimeStorageContext?.loadAuxiliaryFile === 'function'
|
|
914
|
+
? (path) => runtimeStorageContext.loadAuxiliaryFile(path)
|
|
915
|
+
: null,
|
|
898
916
|
});
|
|
899
917
|
if (isRDRRManifest(manifest)) {
|
|
900
918
|
dopplerLoader.setManifest(manifest);
|
|
@@ -942,6 +960,8 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
942
960
|
layerRouterWeights.set(l, {
|
|
943
961
|
weight: weights.routerWeight,
|
|
944
962
|
bias: weights.routerBias || null,
|
|
963
|
+
scale: weights.routerScale || null,
|
|
964
|
+
perExpertScale: weights.routerPerExpertScale || null,
|
|
945
965
|
});
|
|
946
966
|
}
|
|
947
967
|
}
|
|
@@ -954,8 +974,12 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
954
974
|
lmHead: dopplerLoader.lmHead,
|
|
955
975
|
finalNorm: dopplerLoader.finalNorm,
|
|
956
976
|
embeddingPostprocessor: dopplerLoader.embeddingPostprocessor,
|
|
977
|
+
diffusionGemmaSelfConditioning: dopplerLoader.diffusionGemmaSelfConditioning,
|
|
957
978
|
perLayerInputWeights: dopplerLoader.perLayerInputWeights,
|
|
958
979
|
layerRouterWeights,
|
|
980
|
+
loadTiming: typeof dopplerLoader.getLoadTiming === 'function'
|
|
981
|
+
? dopplerLoader.getLoadTiming()
|
|
982
|
+
: null,
|
|
959
983
|
};
|
|
960
984
|
}
|
|
961
985
|
|
|
@@ -978,7 +1002,12 @@ export function initMoERouter(modelConfig, moeRoutingConfig, layerWeights) {
|
|
|
978
1002
|
for (let l = 0; l < modelConfig.numLayers; l++) {
|
|
979
1003
|
const weights = layerWeights.get(`layer_${l}`);
|
|
980
1004
|
if (weights?.routerWeight) {
|
|
981
|
-
router.loadWeights(
|
|
1005
|
+
router.loadWeights(
|
|
1006
|
+
weights.routerWeight,
|
|
1007
|
+
weights.routerBias || null,
|
|
1008
|
+
weights.routerScale || null,
|
|
1009
|
+
weights.routerPerExpertScale || null
|
|
1010
|
+
);
|
|
982
1011
|
log.debug('Pipeline', `Loaded MoE router from layer ${l}${weights.routerBias ? ' (with bias)' : ''}`);
|
|
983
1012
|
break;
|
|
984
1013
|
}
|
|
@@ -24,12 +24,36 @@ import {
|
|
|
24
24
|
hasPerLayerInputBlock,
|
|
25
25
|
applyPerLayerInputBlock,
|
|
26
26
|
applyLayerScalar,
|
|
27
|
+
resolveLayerScalarValue,
|
|
27
28
|
} from './layer.js';
|
|
28
29
|
|
|
29
30
|
// ============================================================================
|
|
30
31
|
// Configurable Layer Pipeline (JSON-Driven)
|
|
31
32
|
// ============================================================================
|
|
32
33
|
|
|
34
|
+
function isFinalResidualOutputStep(steps, stepIndex) {
|
|
35
|
+
const step = steps[stepIndex];
|
|
36
|
+
return step?.op === 'residual_add'
|
|
37
|
+
&& step.dst === 'state'
|
|
38
|
+
&& !step.probeStage
|
|
39
|
+
&& stepIndex === steps.length - 1;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function resolveResidualOutputScaleForPlan(steps, stepIndex, config, context, layerWeights, residualDtype) {
|
|
43
|
+
if (!isFinalResidualOutputStep(steps, stepIndex)) {
|
|
44
|
+
return 1;
|
|
45
|
+
}
|
|
46
|
+
if (hasPerLayerInputBlock(config)) {
|
|
47
|
+
return 1;
|
|
48
|
+
}
|
|
49
|
+
if (residualDtype !== resolveActivationDtype(context.activationDtype)) {
|
|
50
|
+
return 1;
|
|
51
|
+
}
|
|
52
|
+
if (context.debugProbes?.length) {
|
|
53
|
+
return 1;
|
|
54
|
+
}
|
|
55
|
+
return resolveLayerScalarValue(layerWeights?.layerScalar ?? null);
|
|
56
|
+
}
|
|
33
57
|
|
|
34
58
|
function resolveNormWeightForPlan(weight, layerWeights) {
|
|
35
59
|
if (!layerWeights) return null;
|
|
@@ -60,7 +84,7 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
|
|
|
60
84
|
|
|
61
85
|
const planSteps = getLayerPlanSteps(context.pipelinePlan, layerIdx);
|
|
62
86
|
const steps = filterLayerPlanStepsByPhase(planSteps, isPrefill);
|
|
63
|
-
const device = getDevice();
|
|
87
|
+
const device = recorder?.device ?? getDevice();
|
|
64
88
|
if (!device) throw new Error('No GPU device available');
|
|
65
89
|
|
|
66
90
|
const layerType = config.layerTypes?.[layerIdx];
|
|
@@ -194,7 +218,9 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
|
|
|
194
218
|
};
|
|
195
219
|
|
|
196
220
|
try {
|
|
197
|
-
|
|
221
|
+
let layerScalarFused = false;
|
|
222
|
+
for (let stepIndex = 0; stepIndex < steps.length; stepIndex++) {
|
|
223
|
+
const step = steps[stepIndex];
|
|
198
224
|
switch (step.op) {
|
|
199
225
|
case 'save': {
|
|
200
226
|
const src = getSlot(step.src);
|
|
@@ -239,6 +265,7 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
|
|
|
239
265
|
attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
|
|
240
266
|
queryPreAttnScalar: config.queryPreAttnScalar,
|
|
241
267
|
queryKeyNorm: config.queryKeyNorm,
|
|
268
|
+
queryKeyNormWeightLayers: config.queryKeyNormWeightLayers,
|
|
242
269
|
valueNorm: config.valueNorm,
|
|
243
270
|
attentionOutputGate: config.attentionOutputGate,
|
|
244
271
|
outputGateType: config.outputGateType ?? null,
|
|
@@ -265,8 +292,8 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
|
|
|
265
292
|
attnState,
|
|
266
293
|
context.debug,
|
|
267
294
|
{ debugLayers: context.debugLayers },
|
|
268
|
-
(weight, label) => getWeightBuffer(weight, label),
|
|
269
|
-
(weight, label) => getNormWeightBuffer(weight, label, weightConfig, debugFlags),
|
|
295
|
+
(weight, label) => getWeightBuffer(weight, label, device),
|
|
296
|
+
(weight, label) => getNormWeightBuffer(weight, label, weightConfig, debugFlags, device),
|
|
270
297
|
context.debugCheckBuffer,
|
|
271
298
|
recorder,
|
|
272
299
|
context.lora
|
|
@@ -304,9 +331,9 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
|
|
|
304
331
|
|
|
305
332
|
const outputTensor = await doConv(
|
|
306
333
|
srcTensor,
|
|
307
|
-
getWeightBuffer(convInProj, `L${layerIdx}.plan_conv_in_proj
|
|
308
|
-
convKernel ? getWeightBuffer(convKernel, `L${layerIdx}.plan_conv_kernel
|
|
309
|
-
getWeightBuffer(convOutProj, `L${layerIdx}.plan_conv_out_proj
|
|
334
|
+
getWeightBuffer(convInProj, `L${layerIdx}.plan_conv_in_proj`, device),
|
|
335
|
+
convKernel ? getWeightBuffer(convKernel, `L${layerIdx}.plan_conv_kernel`, device) : null,
|
|
336
|
+
getWeightBuffer(convOutProj, `L${layerIdx}.plan_conv_out_proj`, device),
|
|
310
337
|
{
|
|
311
338
|
numTokens,
|
|
312
339
|
hiddenSize,
|
|
@@ -340,7 +367,7 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
|
|
|
340
367
|
if (!weight) {
|
|
341
368
|
throw new Error(`Layer pipeline rmsnorm missing weights for "${step.weight}" at L${layerIdx}`);
|
|
342
369
|
}
|
|
343
|
-
const normWeightBuf = getNormWeightBuffer(weight, `rmsnorm_${step.weight}`, weightConfig, debugFlags);
|
|
370
|
+
const normWeightBuf = getNormWeightBuffer(weight, `rmsnorm_${step.weight}`, weightConfig, debugFlags, device);
|
|
344
371
|
const residualBuf = step.residual ? getSlot(step.residual) : null;
|
|
345
372
|
|
|
346
373
|
const activationDtype = resolveStepInputDtype(step, step.src);
|
|
@@ -388,6 +415,13 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
|
|
|
388
415
|
let outputTensor;
|
|
389
416
|
const { runMoEFFNGPU, runDenseFFNGPU } = await import('./ffn/index.js');
|
|
390
417
|
|
|
418
|
+
if (config.ffnBranchMode === 'dense_plus_moe') {
|
|
419
|
+
throw new Error(
|
|
420
|
+
`Layer ${layerIdx} uses ffn.branchMode="dense_plus_moe", but execution-v1 generic ffn steps ` +
|
|
421
|
+
'cannot express separate dense, expert, and router inputs. Use the sandwich FFN runtime path or add a dedicated plan op.'
|
|
422
|
+
);
|
|
423
|
+
}
|
|
424
|
+
|
|
391
425
|
const canAutoMoe = config.useMoE && isMoELayer(layerIdx, config);
|
|
392
426
|
const useMoe = selectRuleValue(
|
|
393
427
|
'inference',
|
|
@@ -422,11 +456,23 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
|
|
|
422
456
|
const activationDtype = resolveStepInputDtype(step, step.a ?? 'state');
|
|
423
457
|
const aTensor = createTensor(aBuf, activationDtype, [numTokens, hiddenSize], 'plan_residual_a');
|
|
424
458
|
const bTensor = createTensor(bBuf, activationDtype, [numTokens, hiddenSize], 'plan_residual_b');
|
|
459
|
+
const outputScale = resolveResidualOutputScaleForPlan(
|
|
460
|
+
steps,
|
|
461
|
+
stepIndex,
|
|
462
|
+
config,
|
|
463
|
+
context,
|
|
464
|
+
layerWeights,
|
|
465
|
+
activationDtype
|
|
466
|
+
);
|
|
425
467
|
const outputTensor = await doResidualAdd(aTensor, bTensor, size, recorder, {
|
|
426
468
|
label: `L${layerIdx}.residual_add`,
|
|
427
469
|
layerIdx,
|
|
470
|
+
outputScale,
|
|
428
471
|
executionPolicies: context.executionPolicies ?? null,
|
|
429
472
|
});
|
|
473
|
+
if (outputScale !== 1) {
|
|
474
|
+
layerScalarFused = true;
|
|
475
|
+
}
|
|
430
476
|
const outputDtype = resolveStepOutputDtype(step, resolveActivationDtype(outputTensor.dtype));
|
|
431
477
|
setSlot(step.dst, outputTensor.buffer, outputDtype);
|
|
432
478
|
if (step.probeStage) {
|
|
@@ -499,12 +545,14 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
|
|
|
499
545
|
const outputDtype = resolveActivationDtype(outputTensor.dtype);
|
|
500
546
|
setSlot('state', outputTensor.buffer, outputDtype);
|
|
501
547
|
}
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
548
|
+
if (!layerScalarFused) {
|
|
549
|
+
const stateBuffer = getSlot('state');
|
|
550
|
+
const stateDtype = getSlotDtype('state') ?? activationDtype;
|
|
551
|
+
const stateTensor = createTensor(stateBuffer, stateDtype, [numTokens, hiddenSize], 'plan_state_layer_scalar');
|
|
552
|
+
const scaledTensor = await applyLayerScalar(layerIdx, stateTensor, size, context, layerWeights);
|
|
553
|
+
if (scaledTensor.buffer !== stateTensor.buffer) {
|
|
554
|
+
setSlot('state', scaledTensor.buffer, resolveActivationDtype(scaledTensor.dtype));
|
|
555
|
+
}
|
|
508
556
|
}
|
|
509
557
|
} catch (err) {
|
|
510
558
|
cleanupSlots();
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
3
|
import { log, trace } from '../../../debug/index.js';
|
|
4
|
+
import { getRuntimeConfig } from '../../../config/runtime.js';
|
|
4
5
|
import { getDevice } from '../../../gpu/device.js';
|
|
5
6
|
import { releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
|
|
6
7
|
import { allowReadback } from '../../../gpu/perf-guards.js';
|
|
7
8
|
import { createTensor } from '../../../gpu/tensor.js';
|
|
8
9
|
import { recordScale, runScale } from '../../../gpu/kernel-selector.js';
|
|
9
10
|
import {
|
|
10
|
-
doAttention, doRMSNorm, doResidualAdd, doMatmul, doGeLU,
|
|
11
|
+
doAttention, doRMSNorm, doSandwichRMSNormPair, doResidualAdd, doMatmul, doGeLU,
|
|
11
12
|
doConv,
|
|
12
13
|
doCast,
|
|
13
14
|
releaseOrTrack
|
|
@@ -21,6 +22,7 @@ import { logLayer, logAttn, getBufferStats, isKernelDebugEnabled, dumpTokenVecto
|
|
|
21
22
|
import { runProbes } from './probes.js';
|
|
22
23
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
23
24
|
import { recordCheckFiniteness } from '../../../gpu/kernels/check-finiteness.js';
|
|
25
|
+
import { RMSNORM_PAIR_CACHE_LIMIT } from '../../../gpu/kernel-selector.js';
|
|
24
26
|
import { shouldRunFinitenessGuard } from './finiteness-policy.js';
|
|
25
27
|
import { runLinearAttentionLayer } from './linear-attention.js';
|
|
26
28
|
import { validateAttnConfig } from './attention/attn-config.js';
|
|
@@ -46,6 +48,122 @@ export function detectSandwichNorm(config) {
|
|
|
46
48
|
};
|
|
47
49
|
}
|
|
48
50
|
|
|
51
|
+
function shouldUseSandwichRMSNormPairFusion({
|
|
52
|
+
context,
|
|
53
|
+
sandwichNorm,
|
|
54
|
+
layerWeights,
|
|
55
|
+
numTokens,
|
|
56
|
+
hiddenSize,
|
|
57
|
+
attnOutput,
|
|
58
|
+
inputTensor,
|
|
59
|
+
}) {
|
|
60
|
+
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
61
|
+
if (mergedSession?.useSandwichRMSNormPairFusion !== true) {
|
|
62
|
+
return false;
|
|
63
|
+
}
|
|
64
|
+
if (numTokens !== 1) {
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
67
|
+
if (!sandwichNorm.useSandwichNorm || !sandwichNorm.hasPostAttentionNorm || !sandwichNorm.hasPreFeedforwardNorm) {
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
if (!layerWeights?.postAttentionNorm || !layerWeights?.preFeedforwardNorm) {
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
73
|
+
if (attnOutput?.dtype !== 'f32' || (inputTensor && inputTensor.dtype !== 'f32')) {
|
|
74
|
+
throw new Error(
|
|
75
|
+
'useSandwichRMSNormPairFusion requires f32 attention and residual tensors ' +
|
|
76
|
+
`(attn=${String(attnOutput?.dtype)}, residual=${String(inputTensor?.dtype)}).`
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
if (!Number.isInteger(hiddenSize) || hiddenSize <= 0 || hiddenSize > RMSNORM_PAIR_CACHE_LIMIT) {
|
|
80
|
+
throw new Error(
|
|
81
|
+
`useSandwichRMSNormPairFusion requires hiddenSize in 1..${RMSNORM_PAIR_CACHE_LIMIT}; got ${String(hiddenSize)}.`
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
return true;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function releasePrecomputedInputNorm(context, recorder) {
|
|
88
|
+
const precomputed = context.__precomputedInputNorm ?? null;
|
|
89
|
+
context.__precomputedInputNorm = null;
|
|
90
|
+
const buffer = precomputed?.tensor?.buffer ?? null;
|
|
91
|
+
if (buffer) {
|
|
92
|
+
releaseOrTrack(recorder, buffer, context.decodeBuffers);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function takePrecomputedInputNorm(context, layerIdx, recorder) {
|
|
97
|
+
const precomputed = context.__precomputedInputNorm ?? null;
|
|
98
|
+
if (!precomputed) {
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
context.__precomputedInputNorm = null;
|
|
102
|
+
if (precomputed.layerIdx !== layerIdx) {
|
|
103
|
+
const buffer = precomputed?.tensor?.buffer ?? null;
|
|
104
|
+
if (buffer) {
|
|
105
|
+
releaseOrTrack(recorder, buffer, context.decodeBuffers);
|
|
106
|
+
}
|
|
107
|
+
throw new Error(
|
|
108
|
+
`Layer ${layerIdx} received stale precomputed input norm for layer ${String(precomputed.layerIdx)}.`
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
return precomputed.tensor;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function shouldUsePostFfnNextInputRMSNormPairFusion({
|
|
115
|
+
context,
|
|
116
|
+
config,
|
|
117
|
+
sandwichNorm,
|
|
118
|
+
layerIdx,
|
|
119
|
+
layerWeights,
|
|
120
|
+
nextLayerWeights,
|
|
121
|
+
numTokens,
|
|
122
|
+
hiddenSize,
|
|
123
|
+
activationDtype,
|
|
124
|
+
layerScalar,
|
|
125
|
+
}) {
|
|
126
|
+
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
127
|
+
if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
if (numTokens !== 1 || context.phase !== 'decode' || context.diffusionGemmaDecoder === true) {
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
if (context.debug === true || context.debugProbes?.length || context.operatorDiagnostics?.enabled === true) {
|
|
134
|
+
return false;
|
|
135
|
+
}
|
|
136
|
+
if (!context.decodeBuffers || context.pipelinePlan || hasPerLayerInputBlock(config)) {
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
if (layerScalar !== 1 || activationDtype !== 'f32') {
|
|
140
|
+
return false;
|
|
141
|
+
}
|
|
142
|
+
const nextLayerIdx = layerIdx + 1;
|
|
143
|
+
if (nextLayerIdx >= config.numLayers) {
|
|
144
|
+
return false;
|
|
145
|
+
}
|
|
146
|
+
const nextLayerType = config.layerTypes?.[nextLayerIdx];
|
|
147
|
+
if (isConvLayerType(nextLayerType) || isLinearLayerType(nextLayerType)) {
|
|
148
|
+
return false;
|
|
149
|
+
}
|
|
150
|
+
if (!sandwichNorm.useSandwichNorm || !sandwichNorm.hasPostFeedforwardNorm) {
|
|
151
|
+
return false;
|
|
152
|
+
}
|
|
153
|
+
if (!layerWeights?.postFeedforwardNorm || !nextLayerWeights?.inputNorm) {
|
|
154
|
+
return false;
|
|
155
|
+
}
|
|
156
|
+
if (config.useMoE && isMoELayer(layerIdx, config)) {
|
|
157
|
+
return false;
|
|
158
|
+
}
|
|
159
|
+
if (!Number.isInteger(hiddenSize) || hiddenSize <= 0 || hiddenSize > RMSNORM_PAIR_CACHE_LIMIT) {
|
|
160
|
+
throw new Error(
|
|
161
|
+
`usePostFfnNextInputRMSNormPairFusion requires hiddenSize in 1..${RMSNORM_PAIR_CACHE_LIMIT}; got ${String(hiddenSize)}.`
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
return true;
|
|
165
|
+
}
|
|
166
|
+
|
|
49
167
|
|
|
50
168
|
export function isMoELayer(layerIdx, config) {
|
|
51
169
|
if (!config.useMoE) return false;
|
|
@@ -504,10 +622,10 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
504
622
|
// Debug entry (uses debug-utils)
|
|
505
623
|
logLayer(layerIdx, 'enter', isPrefill, { numTokens });
|
|
506
624
|
|
|
507
|
-
const
|
|
625
|
+
const { config, weights, weightConfig, debugFlags, kvCache, ropeFreqsCos, ropeFreqsSin, recorder } = context;
|
|
626
|
+
const device = recorder?.device ?? getDevice();
|
|
508
627
|
if (!device) throw new Error('No GPU device available');
|
|
509
628
|
|
|
510
|
-
const { config, weights, weightConfig, debugFlags, kvCache, ropeFreqsCos, ropeFreqsSin, recorder } = context;
|
|
511
629
|
assertSupportedLayerRuntime(layerIdx, config);
|
|
512
630
|
const { hiddenSize, numHeads, numKVHeads, headDim, rmsNormEps } = config;
|
|
513
631
|
|
|
@@ -522,6 +640,16 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
522
640
|
const sandwichNorm = detectSandwichNorm(config);
|
|
523
641
|
const lastTokenIdx = Math.max(0, numTokens - 1);
|
|
524
642
|
|
|
643
|
+
await runProbes('layer_in', inputBuffer, {
|
|
644
|
+
layerIdx,
|
|
645
|
+
numTokens,
|
|
646
|
+
hiddenSize,
|
|
647
|
+
probes: context.debugProbes,
|
|
648
|
+
recorder,
|
|
649
|
+
operatorDiagnostics: context.operatorDiagnostics,
|
|
650
|
+
dtype: inputTensor.dtype,
|
|
651
|
+
});
|
|
652
|
+
|
|
525
653
|
if (context.pipelinePlan) {
|
|
526
654
|
return processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill, size, context, layerWeights, sandwichNorm);
|
|
527
655
|
}
|
|
@@ -615,7 +743,11 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
615
743
|
let attentionHeadDim = resolveAttentionHeadDim(config, layerType);
|
|
616
744
|
let attentionNumKVHeads = resolveAttentionNumKVHeads(config, layerType, layerWeights, attentionHeadDim);
|
|
617
745
|
let disableRoPE = false;
|
|
618
|
-
let queryKeyNorm = config.queryKeyNorm;
|
|
746
|
+
let queryKeyNorm = config.queryKeyNorm === true;
|
|
747
|
+
const diffusionGemmaDecoder = context.diffusionGemmaDecoder === true;
|
|
748
|
+
if (queryKeyNorm && Array.isArray(config.queryKeyNormLayers)) {
|
|
749
|
+
queryKeyNorm = config.queryKeyNormLayers.includes(layerIdx);
|
|
750
|
+
}
|
|
619
751
|
const { sharedKVSourceLayerIdx, storeSharedKV } = resolveAttentionKVSharing(config, layerIdx, layerType);
|
|
620
752
|
|
|
621
753
|
const attnConfig = {
|
|
@@ -637,11 +769,12 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
637
769
|
attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
|
|
638
770
|
queryPreAttnScalar: config.queryPreAttnScalar,
|
|
639
771
|
queryKeyNorm,
|
|
772
|
+
queryKeyNormWeightLayers: config.queryKeyNormWeightLayers,
|
|
640
773
|
valueNorm: config.valueNorm,
|
|
641
774
|
attentionOutputGate: config.attentionOutputGate,
|
|
642
775
|
outputGateType: config.outputGateType ?? null,
|
|
643
|
-
causalAttention: config.causalAttention,
|
|
644
|
-
multimodalBidirectionalSpan: isSlidingLayerType(layerType)
|
|
776
|
+
causalAttention: diffusionGemmaDecoder ? false : config.causalAttention,
|
|
777
|
+
multimodalBidirectionalSpan: !diffusionGemmaDecoder && isSlidingLayerType(layerType)
|
|
645
778
|
? (context.multimodalBidirectionalSpan ?? null)
|
|
646
779
|
: null,
|
|
647
780
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
@@ -654,9 +787,11 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
654
787
|
disableRoPE,
|
|
655
788
|
sharedKVSourceLayerIdx,
|
|
656
789
|
storeSharedKV,
|
|
790
|
+
diffusionGemmaDecoder,
|
|
657
791
|
};
|
|
658
792
|
|
|
659
793
|
validateAttnConfig(attnConfig, `L${layerIdx}`);
|
|
794
|
+
attnConfig.precomputedInputNorm = takePrecomputedInputNorm(context, layerIdx, recorder);
|
|
660
795
|
|
|
661
796
|
const attnState = {
|
|
662
797
|
ropeFreqsCos: (isLocalLayer && context.ropeLocalCos)
|
|
@@ -738,9 +873,37 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
738
873
|
|
|
739
874
|
// 2. Handle residual connection based on architecture
|
|
740
875
|
|
|
876
|
+
let precomputedFfnInput = null;
|
|
741
877
|
if (residualFused) {
|
|
742
878
|
postAttn = attnOutput;
|
|
743
|
-
if (
|
|
879
|
+
if (shouldUseSandwichRMSNormPairFusion({
|
|
880
|
+
context,
|
|
881
|
+
sandwichNorm,
|
|
882
|
+
layerWeights,
|
|
883
|
+
numTokens,
|
|
884
|
+
hiddenSize,
|
|
885
|
+
attnOutput,
|
|
886
|
+
inputTensor: null,
|
|
887
|
+
})) {
|
|
888
|
+
const postNormWeightBuf = getNormWeightBuffer(layerWeights.postAttentionNorm, 'post_attention_norm', weightConfig, debugFlags);
|
|
889
|
+
const preNormWeightBuf = getNormWeightBuffer(layerWeights.preFeedforwardNorm, 'pre_feedforward_norm', weightConfig, debugFlags);
|
|
890
|
+
const pair = await doSandwichRMSNormPair(attnOutput, null, postNormWeightBuf, preNormWeightBuf, rmsNormEps, {
|
|
891
|
+
batchSize: numTokens,
|
|
892
|
+
hiddenSize,
|
|
893
|
+
label: `L${layerIdx}.post_attn_pre_ffn_norm`,
|
|
894
|
+
layerIdx,
|
|
895
|
+
rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
|
|
896
|
+
}, recorder);
|
|
897
|
+
postAttn = pair.postAttn;
|
|
898
|
+
precomputedFfnInput = pair.ffnInput;
|
|
899
|
+
if (!isGpuBufferInstance(layerWeights.postAttentionNorm) && !isWeightBuffer(layerWeights.postAttentionNorm)) releaseOrTrack(recorder, postNormWeightBuf);
|
|
900
|
+
if (!isGpuBufferInstance(layerWeights.preFeedforwardNorm) && !isWeightBuffer(layerWeights.preFeedforwardNorm)) releaseOrTrack(recorder, preNormWeightBuf);
|
|
901
|
+
if (recorder) {
|
|
902
|
+
recorder.trackTemporaryBuffer(attnOutput.buffer);
|
|
903
|
+
} else {
|
|
904
|
+
releaseBuffer(attnOutput.buffer);
|
|
905
|
+
}
|
|
906
|
+
} else if (sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm && layerWeights?.postAttentionNorm) {
|
|
744
907
|
const normWeightBuf = getNormWeightBuffer(layerWeights.postAttentionNorm, 'post_attention_norm', weightConfig, debugFlags);
|
|
745
908
|
postAttn = await doRMSNorm(attnOutput, normWeightBuf, rmsNormEps, {
|
|
746
909
|
batchSize: numTokens,
|
|
@@ -758,21 +921,52 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
758
921
|
}
|
|
759
922
|
} else if (sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm && layerWeights?.postAttentionNorm) {
|
|
760
923
|
const normWeightBuf = getNormWeightBuffer(layerWeights.postAttentionNorm, 'post_attention_norm', weightConfig, debugFlags);
|
|
761
|
-
|
|
762
|
-
|
|
924
|
+
if (shouldUseSandwichRMSNormPairFusion({
|
|
925
|
+
context,
|
|
926
|
+
sandwichNorm,
|
|
927
|
+
layerWeights,
|
|
928
|
+
numTokens,
|
|
763
929
|
hiddenSize,
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
930
|
+
attnOutput,
|
|
931
|
+
inputTensor,
|
|
932
|
+
})) {
|
|
933
|
+
const preNormWeightBuf = getNormWeightBuffer(layerWeights.preFeedforwardNorm, 'pre_feedforward_norm', weightConfig, debugFlags);
|
|
934
|
+
const pair = await doSandwichRMSNormPair(attnOutput, inputTensor, normWeightBuf, preNormWeightBuf, rmsNormEps, {
|
|
935
|
+
batchSize: numTokens,
|
|
936
|
+
hiddenSize,
|
|
937
|
+
label: `L${layerIdx}.post_attn_pre_ffn_norm`,
|
|
938
|
+
layerIdx,
|
|
939
|
+
rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
|
|
940
|
+
}, recorder);
|
|
941
|
+
postAttn = pair.postAttn;
|
|
942
|
+
precomputedFfnInput = pair.ffnInput;
|
|
943
|
+
if (!isGpuBufferInstance(layerWeights.preFeedforwardNorm) && !isWeightBuffer(layerWeights.preFeedforwardNorm)) releaseOrTrack(recorder, preNormWeightBuf);
|
|
944
|
+
} else if (attnOutput.dtype === inputTensor.dtype) {
|
|
945
|
+
postAttn = await doRMSNorm(attnOutput, normWeightBuf, rmsNormEps, {
|
|
946
|
+
batchSize: numTokens,
|
|
947
|
+
hiddenSize,
|
|
948
|
+
residual: inputTensor,
|
|
949
|
+
label: `L${layerIdx}.post_attn_norm`,
|
|
950
|
+
layerIdx,
|
|
951
|
+
rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
|
|
952
|
+
}, recorder);
|
|
953
|
+
} else {
|
|
954
|
+
const normalizedAttn = await doRMSNorm(attnOutput, normWeightBuf, rmsNormEps, {
|
|
955
|
+
batchSize: numTokens,
|
|
956
|
+
hiddenSize,
|
|
957
|
+
label: `L${layerIdx}.post_attn_norm`,
|
|
958
|
+
layerIdx,
|
|
959
|
+
rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
|
|
960
|
+
}, recorder);
|
|
961
|
+
postAttn = await doResidualAdd(normalizedAttn, inputTensor, size, recorder, {
|
|
962
|
+
label: `L${layerIdx}.post_attn_residual`,
|
|
963
|
+
layerIdx,
|
|
964
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
965
|
+
});
|
|
966
|
+
releaseOrTrack(recorder, normalizedAttn.buffer, context.decodeBuffers);
|
|
967
|
+
}
|
|
773
968
|
|
|
774
969
|
if (!isGpuBufferInstance(layerWeights.postAttentionNorm) && !isWeightBuffer(layerWeights.postAttentionNorm)) releaseOrTrack(recorder, normWeightBuf);
|
|
775
|
-
releaseOrTrack(recorder, normalizedAttn.buffer, context.decodeBuffers);
|
|
776
970
|
if (recorder) {
|
|
777
971
|
recorder.trackTemporaryBuffer(attnOutput.buffer);
|
|
778
972
|
} else {
|
|
@@ -819,10 +1013,60 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
819
1013
|
// 3. Feed-forward network
|
|
820
1014
|
|
|
821
1015
|
let outputTensor;
|
|
1016
|
+
const layerScalar = resolveLayerScalarValue(layerWeights?.layerScalar ?? null);
|
|
1017
|
+
const requestFfnLayerScalarFusion = layerScalar !== 1
|
|
1018
|
+
&& !hasPerLayerInputBlock(config);
|
|
1019
|
+
const nextLayerIdx = layerIdx + 1;
|
|
1020
|
+
const nextLayerWeights = nextLayerIdx < config.numLayers
|
|
1021
|
+
? weights.get(`layer_${nextLayerIdx}`)
|
|
1022
|
+
: null;
|
|
1023
|
+
const usePostFfnNextInputNormPair = shouldUsePostFfnNextInputRMSNormPairFusion({
|
|
1024
|
+
context,
|
|
1025
|
+
config,
|
|
1026
|
+
sandwichNorm,
|
|
1027
|
+
layerIdx,
|
|
1028
|
+
layerWeights,
|
|
1029
|
+
nextLayerWeights,
|
|
1030
|
+
numTokens,
|
|
1031
|
+
hiddenSize,
|
|
1032
|
+
activationDtype,
|
|
1033
|
+
layerScalar,
|
|
1034
|
+
});
|
|
1035
|
+
let layerScalarFused = false;
|
|
822
1036
|
if (sandwichNorm.useSandwichNorm) {
|
|
823
|
-
|
|
1037
|
+
context.__postFfnNextInputNorm = usePostFfnNextInputNormPair
|
|
1038
|
+
? { layerIdx: nextLayerIdx, weight: nextLayerWeights.inputNorm }
|
|
1039
|
+
: null;
|
|
1040
|
+
try {
|
|
1041
|
+
outputTensor = await processFFNWithSandwichNorm(
|
|
1042
|
+
layerIdx,
|
|
1043
|
+
postAttn,
|
|
1044
|
+
numTokens,
|
|
1045
|
+
size,
|
|
1046
|
+
context,
|
|
1047
|
+
layerWeights,
|
|
1048
|
+
sandwichNorm,
|
|
1049
|
+
requestFfnLayerScalarFusion ? layerScalar : 1,
|
|
1050
|
+
precomputedFfnInput
|
|
1051
|
+
);
|
|
1052
|
+
} finally {
|
|
1053
|
+
context.__postFfnNextInputNorm = null;
|
|
1054
|
+
}
|
|
1055
|
+
layerScalarFused = context.__layerScalarFusedFired === true;
|
|
1056
|
+
context.__layerScalarFusedFired = false;
|
|
824
1057
|
} else {
|
|
825
|
-
outputTensor = await processFFNStandard(
|
|
1058
|
+
outputTensor = await processFFNStandard(
|
|
1059
|
+
layerIdx,
|
|
1060
|
+
postAttn,
|
|
1061
|
+
numTokens,
|
|
1062
|
+
size,
|
|
1063
|
+
context,
|
|
1064
|
+
layerWeights,
|
|
1065
|
+
fusedResidualForFFN,
|
|
1066
|
+
requestFfnLayerScalarFusion ? layerScalar : 1
|
|
1067
|
+
);
|
|
1068
|
+
layerScalarFused = context.__layerScalarFusedFired === true;
|
|
1069
|
+
context.__layerScalarFusedFired = false;
|
|
826
1070
|
}
|
|
827
1071
|
|
|
828
1072
|
// Keep activation dtype consistent across layers. Some FFN paths can emit f32
|
|
@@ -862,10 +1106,12 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
862
1106
|
}
|
|
863
1107
|
|
|
864
1108
|
let finalOutput = outputTensor;
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
1109
|
+
if (!layerScalarFused) {
|
|
1110
|
+
const scaledOutput = await applyLayerScalar(layerIdx, finalOutput, size, context, layerWeights);
|
|
1111
|
+
if (scaledOutput.buffer !== finalOutput.buffer) {
|
|
1112
|
+
releaseOrTrack(recorder, finalOutput.buffer, context.decodeBuffers);
|
|
1113
|
+
finalOutput = scaledOutput;
|
|
1114
|
+
}
|
|
869
1115
|
}
|
|
870
1116
|
await debugLayerTensor(context, layerIdx, 'final layer output', finalOutput, numTokens, hiddenSize);
|
|
871
1117
|
await runProbes('layer_out', finalOutput.buffer, {
|
|
@@ -904,6 +1150,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
904
1150
|
released.add(buf);
|
|
905
1151
|
releaseOrTrack(recorder, buf);
|
|
906
1152
|
};
|
|
1153
|
+
releasePrecomputedInputNorm(context, recorder);
|
|
907
1154
|
if (postAttn?.buffer) releaseOnce(postAttn.buffer);
|
|
908
1155
|
if (attnOutput?.buffer && attnOutput.buffer !== postAttn?.buffer) releaseOnce(attnOutput.buffer);
|
|
909
1156
|
throw error;
|