doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -18,12 +18,20 @@ import { composeTransforms } from '../../../config/transforms/execution-graph-tr
|
|
|
18
18
|
import { resolveRangeAwareSelectiveWideningConfig } from './finiteness-policy.js';
|
|
19
19
|
import { log } from '../../../debug/index.js';
|
|
20
20
|
|
|
21
|
+
const SESSION_CAPABILITY_TRANSFORMS = new Set([
|
|
22
|
+
'disableRetainQ4KMaterialization',
|
|
23
|
+
]);
|
|
24
|
+
|
|
21
25
|
export function hasExecutionV1(manifestInference) {
|
|
22
26
|
return manifestInference?.schema === EXECUTION_V1_SCHEMA_ID
|
|
23
27
|
&& manifestInference?.execution
|
|
24
28
|
&& typeof manifestInference.execution.kernels === 'object';
|
|
25
29
|
}
|
|
26
30
|
|
|
31
|
+
function isSessionCapabilityTransform(name) {
|
|
32
|
+
return SESSION_CAPABILITY_TRANSFORMS.has(name);
|
|
33
|
+
}
|
|
34
|
+
|
|
27
35
|
function mergeExecutionV1Session(manifestSession, runtimeSession) {
|
|
28
36
|
return mergeRuntimeValues(
|
|
29
37
|
manifestSession ?? {},
|
|
@@ -386,6 +394,8 @@ export function compileExecutionV1(options = {}) {
|
|
|
386
394
|
}
|
|
387
395
|
|
|
388
396
|
const activationDtype = session.compute.defaults.activationDtype;
|
|
397
|
+
const mathDtype = session.compute.defaults.mathDtype ?? null;
|
|
398
|
+
const accumDtype = session.compute.defaults.accumDtype ?? null;
|
|
389
399
|
const kvDtype = session?.kvcache?.kvDtype ?? activationDtype;
|
|
390
400
|
const layerTypes = manifestInference?.layerPattern?.layerTypes ?? null;
|
|
391
401
|
const finitenessPolicy = resolveRangeAwareSelectiveWideningConfig(runtimeCompute);
|
|
@@ -404,10 +414,13 @@ export function compileExecutionV1(options = {}) {
|
|
|
404
414
|
if (capabilities) {
|
|
405
415
|
const graphContext = {
|
|
406
416
|
activationDtype,
|
|
417
|
+
mathDtype,
|
|
418
|
+
accumDtype,
|
|
407
419
|
kvDtype,
|
|
408
420
|
headDim,
|
|
409
421
|
modelId,
|
|
410
422
|
layerTypes,
|
|
423
|
+
retainQ4KMaterialization: session?.retainQ4KMaterialization === true,
|
|
411
424
|
...summarizeExecutionGraphContext(execution),
|
|
412
425
|
};
|
|
413
426
|
const resolved = resolveCapabilityTransforms(capabilities, platform, graphContext);
|
|
@@ -424,22 +437,35 @@ export function compileExecutionV1(options = {}) {
|
|
|
424
437
|
'Use capability-aware remap for manifest-owned execution graphs or choose a compatible runtime.'
|
|
425
438
|
);
|
|
426
439
|
}
|
|
427
|
-
const
|
|
428
|
-
|
|
429
|
-
capabilities,
|
|
430
|
-
platform: platform ?? { id: 'unknown', vendor: 'unknown', architecture: 'unknown' },
|
|
431
|
-
activationDtype,
|
|
432
|
-
kvDtype,
|
|
433
|
-
modelId,
|
|
434
|
-
layerTypes,
|
|
440
|
+
const graphTransforms = resolved.transforms.filter((_, index) => {
|
|
441
|
+
return !isSessionCapabilityTransform(resolved.names[index]);
|
|
435
442
|
});
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
443
|
+
const sessionTransformNames = resolved.names.filter(isSessionCapabilityTransform);
|
|
444
|
+
if (graphTransforms.length > 0) {
|
|
445
|
+
const composed = composeTransforms(...graphTransforms);
|
|
446
|
+
const transformed = composed(execution, {
|
|
447
|
+
capabilities,
|
|
448
|
+
platform: platform ?? { id: 'unknown', vendor: 'unknown', architecture: 'unknown' },
|
|
449
|
+
activationDtype,
|
|
450
|
+
mathDtype,
|
|
451
|
+
accumDtype,
|
|
452
|
+
kvDtype,
|
|
453
|
+
modelId,
|
|
454
|
+
layerTypes,
|
|
455
|
+
});
|
|
456
|
+
if (transformed !== execution) {
|
|
457
|
+
execution = transformed;
|
|
458
|
+
graphWasTransformed = true;
|
|
459
|
+
appliedTransformNames.push(...resolved.names.filter((name) => !isSessionCapabilityTransform(name)));
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
if (sessionTransformNames.length > 0) {
|
|
463
|
+
appliedTransformNames.push(...sessionTransformNames);
|
|
464
|
+
}
|
|
465
|
+
if (appliedTransformNames.length > 0) {
|
|
440
466
|
log.info(
|
|
441
467
|
'ExecutionV1',
|
|
442
|
-
`Capability transforms applied: [${
|
|
468
|
+
`Capability transforms applied: [${appliedTransformNames.join(', ')}] (${resolved.reason})`
|
|
443
469
|
);
|
|
444
470
|
}
|
|
445
471
|
} else {
|
|
@@ -456,6 +482,8 @@ export function compileExecutionV1(options = {}) {
|
|
|
456
482
|
capabilities,
|
|
457
483
|
platform: platform ?? { id: 'unknown', vendor: 'unknown', architecture: 'unknown' },
|
|
458
484
|
activationDtype,
|
|
485
|
+
mathDtype,
|
|
486
|
+
accumDtype,
|
|
459
487
|
kvDtype,
|
|
460
488
|
headDim,
|
|
461
489
|
modelId,
|
|
@@ -486,25 +514,33 @@ export function compileExecutionV1(options = {}) {
|
|
|
486
514
|
// KV cache and all compute dtypes must also be f32.
|
|
487
515
|
const activationWidened = appliedTransformNames.includes('widenToF32Activations');
|
|
488
516
|
const fullF32 = activationWidened && capabilities?.hasF16 === false;
|
|
489
|
-
const
|
|
490
|
-
|
|
517
|
+
const useSelectiveF16Primary =
|
|
518
|
+
(
|
|
519
|
+
appliedTransformNames.includes('useQwenF16PrimaryMatmuls')
|
|
520
|
+
|| appliedTransformNames.includes('useGemma4Int4PleSelectiveF16Decode')
|
|
521
|
+
)
|
|
491
522
|
&& !appliedTransformNames.includes('narrowToF16Activations');
|
|
523
|
+
const retainQ4KMaterializationDisabled =
|
|
524
|
+
appliedTransformNames.includes('disableRetainQ4KMaterialization');
|
|
492
525
|
let effectiveSession = session;
|
|
493
|
-
if (activationWidened ||
|
|
526
|
+
if (activationWidened || useSelectiveF16Primary || retainQ4KMaterializationDisabled) {
|
|
494
527
|
const f32Defaults = fullF32
|
|
495
528
|
? { activationDtype: 'f32', mathDtype: 'f32', accumDtype: 'f32', outputDtype: 'f32' }
|
|
496
|
-
:
|
|
529
|
+
: useSelectiveF16Primary
|
|
497
530
|
? { activationDtype: 'f32', outputDtype: 'f32' }
|
|
498
531
|
: { activationDtype: 'f32' };
|
|
499
532
|
effectiveSession = {
|
|
500
533
|
...session,
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
...session.compute
|
|
505
|
-
|
|
534
|
+
...(retainQ4KMaterializationDisabled ? { retainQ4KMaterialization: false } : {}),
|
|
535
|
+
...(activationWidened || useSelectiveF16Primary ? {
|
|
536
|
+
compute: {
|
|
537
|
+
...session.compute,
|
|
538
|
+
defaults: {
|
|
539
|
+
...session.compute.defaults,
|
|
540
|
+
...f32Defaults,
|
|
541
|
+
},
|
|
506
542
|
},
|
|
507
|
-
},
|
|
543
|
+
} : {}),
|
|
508
544
|
...(fullF32 && session?.kvcache ? {
|
|
509
545
|
kvcache: {
|
|
510
546
|
...session.kvcache,
|
|
@@ -570,6 +606,37 @@ export function compileExecutionV1(options = {}) {
|
|
|
570
606
|
}
|
|
571
607
|
}
|
|
572
608
|
|
|
609
|
+
// Lane integrity: capture the pre-transform (manifest+profile-agreed) lane
|
|
610
|
+
// and the post-transform (actually dispatched) lane so receipts can honestly
|
|
611
|
+
// surface a capability-driven widening. The manifest-binding gate ensures
|
|
612
|
+
// the pre-transform session already matches manifest quantizationInfo.compute;
|
|
613
|
+
// any drift here is owned by capability transforms (e.g., widenToF32Activations
|
|
614
|
+
// on hasF16=false GPUs).
|
|
615
|
+
const declaredLane = {
|
|
616
|
+
activationDtype,
|
|
617
|
+
mathDtype,
|
|
618
|
+
accumDtype,
|
|
619
|
+
kvDtype,
|
|
620
|
+
};
|
|
621
|
+
const executedLane = {
|
|
622
|
+
activationDtype: effectiveSession.compute?.defaults?.activationDtype ?? null,
|
|
623
|
+
mathDtype: effectiveSession.compute?.defaults?.mathDtype ?? null,
|
|
624
|
+
accumDtype: effectiveSession.compute?.defaults?.accumDtype ?? null,
|
|
625
|
+
kvDtype: effectiveSession.kvcache?.kvDtype ?? effectiveSession.compute?.defaults?.activationDtype ?? null,
|
|
626
|
+
};
|
|
627
|
+
const laneFieldDelta = (
|
|
628
|
+
declaredLane.activationDtype !== executedLane.activationDtype
|
|
629
|
+
|| declaredLane.mathDtype !== executedLane.mathDtype
|
|
630
|
+
|| declaredLane.accumDtype !== executedLane.accumDtype
|
|
631
|
+
|| declaredLane.kvDtype !== executedLane.kvDtype
|
|
632
|
+
);
|
|
633
|
+
const laneIntegrity = {
|
|
634
|
+
declared: declaredLane,
|
|
635
|
+
executed: executedLane,
|
|
636
|
+
status: laneFieldDelta ? 'transformed' : 'matches',
|
|
637
|
+
transforms: [...appliedTransformNames],
|
|
638
|
+
};
|
|
639
|
+
|
|
573
640
|
const layerPipelineResult = buildLayerPipelineFromExecution(resolvedSteps, {
|
|
574
641
|
logIncompatibleOps: !(kernelPath && inlineKernelPathEnabled),
|
|
575
642
|
ffnDtypeFallback: requireSessionActivationDtype(effectiveSession),
|
|
@@ -600,6 +667,7 @@ export function compileExecutionV1(options = {}) {
|
|
|
600
667
|
...(layerPipeline ? { pipeline: layerPipeline } : {}),
|
|
601
668
|
},
|
|
602
669
|
appliedTransforms: appliedTransformNames,
|
|
670
|
+
laneIntegrity,
|
|
603
671
|
fallbackKernelPath,
|
|
604
672
|
};
|
|
605
673
|
}
|
|
@@ -621,6 +689,7 @@ export function compileExecutionV1(options = {}) {
|
|
|
621
689
|
export function applyExecutionV1RuntimeConfig(options = {}) {
|
|
622
690
|
const runtimeConfig = options.runtimeConfig ?? null;
|
|
623
691
|
const runtimeOverrides = options.runtimeOverrides;
|
|
692
|
+
const hasExplicitRuntimeOverrides = runtimeOverrides != null;
|
|
624
693
|
const manifest = options.manifest ?? null;
|
|
625
694
|
if (!runtimeConfig || !manifest?.inference) {
|
|
626
695
|
return { runtimeConfig, executionV1State: null };
|
|
@@ -644,10 +713,10 @@ export function applyExecutionV1RuntimeConfig(options = {}) {
|
|
|
644
713
|
headDim: options.headDim ?? manifest.architecture?.headDim ?? null,
|
|
645
714
|
capabilities: options.capabilities ?? null,
|
|
646
715
|
platform: options.platform ?? null,
|
|
647
|
-
runtimeSession:
|
|
716
|
+
runtimeSession: hasExplicitRuntimeOverrides
|
|
648
717
|
? resolveRuntimeInferenceOverrideSection(runtimeOverrides, 'session')
|
|
649
718
|
: (runtimeConfig.inference?.session ?? null),
|
|
650
|
-
runtimeCompute:
|
|
719
|
+
runtimeCompute: hasExplicitRuntimeOverrides
|
|
651
720
|
? resolveRuntimeInferenceOverrideSection(runtimeOverrides, 'compute')
|
|
652
721
|
: (runtimeConfig.inference?.compute ?? null),
|
|
653
722
|
kernelPathPolicy: runtimeConfig.inference?.kernelPathPolicy ?? null,
|
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
resolveWeightBufferMaterialization,
|
|
13
13
|
} from '../../../../gpu/weight-buffer.js';
|
|
14
14
|
import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
|
|
15
|
+
import { getRuntimeConfig } from '../../../../config/runtime.js';
|
|
15
16
|
import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
|
|
16
17
|
import {
|
|
17
18
|
runFusedFFN,
|
|
@@ -27,6 +28,7 @@ import { isKernelDebugEnabled, dumpTokenVector } from '../debug-utils.js';
|
|
|
27
28
|
import { applyLoRA } from '../lora-apply.js';
|
|
28
29
|
import { getLoRAModule } from '../lora.js';
|
|
29
30
|
import { getWeightBuffer, getNormWeightBuffer } from '../weights.js';
|
|
31
|
+
import { runProbes } from '../probes.js';
|
|
30
32
|
import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
31
33
|
import {
|
|
32
34
|
getKernelPathMatmulPrecision,
|
|
@@ -145,8 +147,20 @@ export function resolveDenseFFNFusedPathDtypes(options = {}) {
|
|
|
145
147
|
}
|
|
146
148
|
|
|
147
149
|
function hasExplicitMatmulPrecision(role, phase, layerIdx, kernelPath) {
|
|
150
|
+
// Only treat precision as "explicit split" when declared on the role's OWN
|
|
151
|
+
// step. The FUSED_FFN_PRECISION_FALLBACK_ROLES fallback that resolves via
|
|
152
|
+
// the aggregate `ffn` step's precision applies identically to gate/up/down
|
|
153
|
+
// and must NOT force the split path — otherwise a manifest-declared `ffn`
|
|
154
|
+
// precision permanently blocks the fused gate_up_activation kernel.
|
|
148
155
|
const precision = getKernelPathMatmulPrecision(role, phase, layerIdx, kernelPath);
|
|
149
|
-
|
|
156
|
+
if (!precision) return false;
|
|
157
|
+
const fallbackPrecision = getKernelPathMatmulPrecision('ffn', phase, layerIdx, kernelPath);
|
|
158
|
+
const inheritedFromFfn = fallbackPrecision
|
|
159
|
+
&& fallbackPrecision.inputDtype === precision.inputDtype
|
|
160
|
+
&& fallbackPrecision.outputDtype === precision.outputDtype
|
|
161
|
+
&& fallbackPrecision.activationDtype === precision.activationDtype;
|
|
162
|
+
if (inheritedFromFfn) return false;
|
|
163
|
+
return precision.inputDtype != null || precision.outputDtype != null;
|
|
150
164
|
}
|
|
151
165
|
|
|
152
166
|
export function canUseNativeF16FusedGateUp(options = {}) {
|
|
@@ -325,7 +339,7 @@ export async function runDenseFFNGPU(
|
|
|
325
339
|
if (!device) throw new Error('No GPU device');
|
|
326
340
|
|
|
327
341
|
const { config, recorder } = context;
|
|
328
|
-
const { hiddenSize, hiddenActivation, swigluLimit } = config;
|
|
342
|
+
const { hiddenSize, hiddenActivation, swigluLimit, useDoubleWideMlp } = config;
|
|
329
343
|
const intermediateSize = resolveLayerIntermediateSize(config, layerIdx);
|
|
330
344
|
const lastTokenIdx = Math.max(0, numTokens - 1);
|
|
331
345
|
const lora = context.lora || null;
|
|
@@ -434,6 +448,22 @@ export async function runDenseFFNGPU(
|
|
|
434
448
|
releaseBuffer(gateUpOutput.buffer);
|
|
435
449
|
}
|
|
436
450
|
|
|
451
|
+
// Opt-in WideTile+residual fusion: if the caller (processFFNStandard)
|
|
452
|
+
// staged a residual tensor on the context AND no LoRA is present on
|
|
453
|
+
// down_proj (LoRA requires the pre-residual output for its add), route
|
|
454
|
+
// this matmul to the q4_fused_widetile_residual variant which produces
|
|
455
|
+
// (ffn_down_result + residual) in one dispatch. Tell the caller via a
|
|
456
|
+
// context flag so processFFNStandard skips its downstream doResidualAdd.
|
|
457
|
+
const pendingResidual = context.__pendingFfnResidualTensor;
|
|
458
|
+
const downLoraProbe = getLoRAModule(lora, layerIdx, 'down_proj');
|
|
459
|
+
const mergedSession = getRuntimeConfig().inference?.session;
|
|
460
|
+
const tryFuseDownResidual = pendingResidual != null
|
|
461
|
+
&& !downLoraProbe
|
|
462
|
+
&& numTokens > 1
|
|
463
|
+
&& activatedOutput.dtype === 'f32'
|
|
464
|
+
&& downOutputDtype === 'f32'
|
|
465
|
+
&& pendingResidual.dtype === 'f32';
|
|
466
|
+
let residualFusedHere = false;
|
|
437
467
|
let output = await doMatmul(
|
|
438
468
|
activatedOutput, downWeight,
|
|
439
469
|
numTokens, hiddenSize, intermediateSize,
|
|
@@ -445,9 +475,26 @@ export async function runDenseFFNGPU(
|
|
|
445
475
|
outputDtype: downOutputDtype,
|
|
446
476
|
role: 'ffn_down',
|
|
447
477
|
executionPolicies: context.executionPolicies ?? null,
|
|
478
|
+
residualTensor: tryFuseDownResidual ? pendingResidual : null,
|
|
448
479
|
},
|
|
449
480
|
recorder
|
|
450
481
|
);
|
|
482
|
+
// Detect whether the fusion fired: if the selected variant for ffn_down
|
|
483
|
+
// matmul was q4_fused_widetile_residual, then output IS post-residual.
|
|
484
|
+
// We infer this cheaply by re-checking the conditions the selector uses.
|
|
485
|
+
// (A cleaner signal would require a return-shape change across all
|
|
486
|
+
// dense.js paths; this local signal is enough for correctness.)
|
|
487
|
+
{
|
|
488
|
+
if (tryFuseDownResidual
|
|
489
|
+
&& getKernelCapabilities().hasF16 === true
|
|
490
|
+
&& mergedSession?.useWideTileResidualFusion === true
|
|
491
|
+
&& mergedSession?.useWideTileQ4KPrefill === true
|
|
492
|
+
&& mergedSession?.retainQ4KMaterialization === true
|
|
493
|
+
) {
|
|
494
|
+
residualFusedHere = true;
|
|
495
|
+
context.__ffnResidualFusedFired = true;
|
|
496
|
+
}
|
|
497
|
+
}
|
|
451
498
|
|
|
452
499
|
const loraDown = getLoRAModule(lora, layerIdx, 'down_proj');
|
|
453
500
|
if (loraDown) {
|
|
@@ -537,6 +584,7 @@ export async function runDenseFFNGPU(
|
|
|
537
584
|
f16BatchSupported,
|
|
538
585
|
batchSize: numTokens,
|
|
539
586
|
hiddenSizeAligned32,
|
|
587
|
+
useDoubleWideMlp: Boolean(useDoubleWideMlp),
|
|
540
588
|
});
|
|
541
589
|
const useFusedGateUp = gateUpPathMode === 'split'
|
|
542
590
|
? false
|
|
@@ -608,6 +656,16 @@ export async function runDenseFFNGPU(
|
|
|
608
656
|
releaseOrTrack(recorder, isWeightBuffer(upWeight) ? upWeight.buffer : upWeight);
|
|
609
657
|
}
|
|
610
658
|
|
|
659
|
+
// Opt-in WideTile+residual fusion (fused-gate-up path).
|
|
660
|
+
const pendingResidualFused = context.__pendingFfnResidualTensor;
|
|
661
|
+
const downLoraProbeFused = getLoRAModule(lora, layerIdx, 'down_proj');
|
|
662
|
+
const mergedSessionFused = getRuntimeConfig().inference?.session;
|
|
663
|
+
const tryFuseDownResidualFused = pendingResidualFused != null
|
|
664
|
+
&& !downLoraProbeFused
|
|
665
|
+
&& numTokens > 1
|
|
666
|
+
&& downInput.dtype === 'f32'
|
|
667
|
+
&& fusedDownOutputDtype === 'f32'
|
|
668
|
+
&& pendingResidualFused.dtype === 'f32';
|
|
611
669
|
let output = await doMatmul(
|
|
612
670
|
downInput,
|
|
613
671
|
downWeight,
|
|
@@ -622,9 +680,20 @@ export async function runDenseFFNGPU(
|
|
|
622
680
|
outputDtype: fusedDownOutputDtype,
|
|
623
681
|
role: 'ffn_down',
|
|
624
682
|
executionPolicies: context.executionPolicies ?? null,
|
|
683
|
+
residualTensor: tryFuseDownResidualFused ? pendingResidualFused : null,
|
|
625
684
|
},
|
|
626
685
|
recorder
|
|
627
686
|
);
|
|
687
|
+
{
|
|
688
|
+
if (tryFuseDownResidualFused
|
|
689
|
+
&& getKernelCapabilities().hasF16 === true
|
|
690
|
+
&& mergedSessionFused?.useWideTileResidualFusion === true
|
|
691
|
+
&& mergedSessionFused?.useWideTileQ4KPrefill === true
|
|
692
|
+
&& mergedSessionFused?.retainQ4KMaterialization === true
|
|
693
|
+
) {
|
|
694
|
+
context.__ffnResidualFusedFired = true;
|
|
695
|
+
}
|
|
696
|
+
}
|
|
628
697
|
|
|
629
698
|
const loraDown = getLoRAModule(lora, layerIdx, 'down_proj');
|
|
630
699
|
if (loraDown) {
|
|
@@ -748,6 +817,84 @@ export async function runDenseFFNGPU(
|
|
|
748
817
|
});
|
|
749
818
|
sharedInputOwned = sharedInputTensor !== inputTensor;
|
|
750
819
|
}
|
|
820
|
+
// Opt-in fused gate + up + GeGLU path. Replaces 3 separate dispatches
|
|
821
|
+
// (gate_proj + up_proj + gelu activation) with a single fused kernel when
|
|
822
|
+
// preconditions match: prefill (M>1), f16-materialisable weights + f16
|
|
823
|
+
// activations, gelu activation, no LoRA on gate/up. Gated by
|
|
824
|
+
// runtime.inference.session.useFusedGateUpGelu (default false).
|
|
825
|
+
const gateF16 = resolveWeightBufferMaterialization(layerWeights.gate, 'f16');
|
|
826
|
+
const upF16 = resolveWeightBufferMaterialization(layerWeights.up, 'f16');
|
|
827
|
+
const gateF16Dtype = getWeightDtype(gateF16);
|
|
828
|
+
const upF16Dtype = getWeightDtype(upF16);
|
|
829
|
+
const earlyLoraGate = getLoRAModule(lora, layerIdx, 'gate_proj');
|
|
830
|
+
const earlyLoraUp = getLoRAModule(lora, layerIdx, 'up_proj');
|
|
831
|
+
const fusedGateUpGeluCandidate = context.useFusedGateUpGelu === true
|
|
832
|
+
&& numTokens > 1
|
|
833
|
+
&& hiddenActivation === 'gelu'
|
|
834
|
+
&& !earlyLoraGate && !earlyLoraUp
|
|
835
|
+
&& sharedInputTensor.dtype === 'f16'
|
|
836
|
+
&& gateF16Dtype === 'f16'
|
|
837
|
+
&& upF16Dtype === 'f16';
|
|
838
|
+
if (fusedGateUpGeluCandidate) {
|
|
839
|
+
const { runFusedGateUpGelu, recordFusedGateUpGelu } =
|
|
840
|
+
await import('../../../../gpu/kernels/fused-gate-up-gelu.js');
|
|
841
|
+
const fused = recorder
|
|
842
|
+
? await recordFusedGateUpGelu(recorder, sharedInputTensor, gateF16, upF16, {
|
|
843
|
+
M: numTokens,
|
|
844
|
+
hiddenSize,
|
|
845
|
+
intermediateSize,
|
|
846
|
+
transposeB: true,
|
|
847
|
+
})
|
|
848
|
+
: await runFusedGateUpGelu(sharedInputTensor, gateF16, upF16, {
|
|
849
|
+
M: numTokens,
|
|
850
|
+
hiddenSize,
|
|
851
|
+
intermediateSize,
|
|
852
|
+
transposeB: true,
|
|
853
|
+
});
|
|
854
|
+
if (sharedInputOwned) {
|
|
855
|
+
if (recorder) { recorder.trackTemporaryBuffer(sharedInputTensor.buffer); }
|
|
856
|
+
else { releaseBuffer(sharedInputTensor.buffer); }
|
|
857
|
+
}
|
|
858
|
+
// Proceed directly to down_proj with the fused activation output.
|
|
859
|
+
const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
|
|
860
|
+
let downInputTensor = fused;
|
|
861
|
+
let downInputOwned = false;
|
|
862
|
+
if (downInputDtype && fused.dtype !== downInputDtype) {
|
|
863
|
+
downInputTensor = await coerceTensorDtype(fused, downInputDtype, recorder, {
|
|
864
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
865
|
+
op: 'ffn_down_input',
|
|
866
|
+
transitionDeclaredBy: 'step_precision',
|
|
867
|
+
});
|
|
868
|
+
downInputOwned = downInputTensor !== fused;
|
|
869
|
+
}
|
|
870
|
+
let outFused = await doMatmul(
|
|
871
|
+
downInputTensor,
|
|
872
|
+
downWeight,
|
|
873
|
+
numTokens,
|
|
874
|
+
hiddenSize,
|
|
875
|
+
intermediateSize,
|
|
876
|
+
{
|
|
877
|
+
transposeB: 'auto',
|
|
878
|
+
label: `L${layerIdx}.ffn_down`,
|
|
879
|
+
layerIdx,
|
|
880
|
+
kernelPath,
|
|
881
|
+
outputDtype: downOutputDtype,
|
|
882
|
+
role: 'ffn_down',
|
|
883
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
884
|
+
},
|
|
885
|
+
recorder
|
|
886
|
+
);
|
|
887
|
+
if (!isGpuBufferInstance(layerWeights.down) && !isWeightBuffer(layerWeights.down)) {
|
|
888
|
+
releaseOrTrack(recorder, isWeightBuffer(downWeight) ? downWeight.buffer : downWeight);
|
|
889
|
+
}
|
|
890
|
+
if (downInputOwned) {
|
|
891
|
+
if (recorder) { recorder.trackTemporaryBuffer(downInputTensor.buffer); }
|
|
892
|
+
else { releaseBuffer(downInputTensor.buffer); }
|
|
893
|
+
}
|
|
894
|
+
if (recorder) { recorder.trackTemporaryBuffer(fused.buffer); }
|
|
895
|
+
else { releaseBuffer(fused.buffer); }
|
|
896
|
+
return outFused;
|
|
897
|
+
}
|
|
751
898
|
const gateWeight = getWeightBuffer(layerWeights.gate, 'ffn_gate');
|
|
752
899
|
let gateOutput = await doMatmul(
|
|
753
900
|
gateInputDtype === sharedInputTensor.dtype ? sharedInputTensor : inputTensor,
|
|
@@ -849,6 +996,25 @@ export async function runDenseFFNGPU(
|
|
|
849
996
|
});
|
|
850
997
|
}
|
|
851
998
|
|
|
999
|
+
await runProbes('ffn_gate', gateOutput.buffer, {
|
|
1000
|
+
layerIdx,
|
|
1001
|
+
numTokens,
|
|
1002
|
+
hiddenSize: intermediateSize,
|
|
1003
|
+
probes: context.debugProbes,
|
|
1004
|
+
recorder,
|
|
1005
|
+
operatorDiagnostics: context.operatorDiagnostics,
|
|
1006
|
+
dtype: gateOutput.dtype,
|
|
1007
|
+
});
|
|
1008
|
+
await runProbes('ffn_up', upOutput.buffer, {
|
|
1009
|
+
layerIdx,
|
|
1010
|
+
numTokens,
|
|
1011
|
+
hiddenSize: intermediateSize,
|
|
1012
|
+
probes: context.debugProbes,
|
|
1013
|
+
recorder,
|
|
1014
|
+
operatorDiagnostics: context.operatorDiagnostics,
|
|
1015
|
+
dtype: upOutput.dtype,
|
|
1016
|
+
});
|
|
1017
|
+
|
|
852
1018
|
const activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
|
|
853
1019
|
size: numTokens * intermediateSize,
|
|
854
1020
|
gate: gateOutput,
|
|
@@ -866,6 +1032,16 @@ export async function runDenseFFNGPU(
|
|
|
866
1032
|
});
|
|
867
1033
|
}
|
|
868
1034
|
|
|
1035
|
+
await runProbes('ffn_act', activatedOutput.buffer, {
|
|
1036
|
+
layerIdx,
|
|
1037
|
+
numTokens,
|
|
1038
|
+
hiddenSize: intermediateSize,
|
|
1039
|
+
probes: context.debugProbes,
|
|
1040
|
+
recorder,
|
|
1041
|
+
operatorDiagnostics: context.operatorDiagnostics,
|
|
1042
|
+
dtype: activatedOutput.dtype,
|
|
1043
|
+
});
|
|
1044
|
+
|
|
869
1045
|
if (recorder) {
|
|
870
1046
|
recorder.trackTemporaryBuffer(gateOutput.buffer);
|
|
871
1047
|
recorder.trackTemporaryBuffer(upOutput.buffer);
|
|
@@ -975,7 +1151,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
|
|
|
975
1151
|
if (!device) throw new Error('No GPU device');
|
|
976
1152
|
|
|
977
1153
|
const { config, weightConfig, debugFlags, recorder } = context;
|
|
978
|
-
const { hiddenSize, hiddenActivation, swigluLimit } = config;
|
|
1154
|
+
const { hiddenSize, hiddenActivation, swigluLimit, useDoubleWideMlp } = config;
|
|
979
1155
|
const intermediateSize = resolveLayerIntermediateSize(config, layerIdx);
|
|
980
1156
|
const lora = context.lora || null;
|
|
981
1157
|
const ffnStepPrecision = context.ffnStepPrecision ?? null;
|
|
@@ -1124,6 +1300,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
|
|
|
1124
1300
|
f16BatchSupported: getKernelCapabilities().hasF16,
|
|
1125
1301
|
batchSize: numTokens,
|
|
1126
1302
|
hiddenSizeAligned32,
|
|
1303
|
+
useDoubleWideMlp: Boolean(useDoubleWideMlp),
|
|
1127
1304
|
});
|
|
1128
1305
|
const canUseFusedGateUp = gateUpPathMode === 'split'
|
|
1129
1306
|
? false
|
|
@@ -77,6 +77,18 @@ export async function processFFNStandard(
|
|
|
77
77
|
await debugFFNBuffer(context, layerIdx, 'FFN input', normedTensor, numTokens, hiddenSize);
|
|
78
78
|
|
|
79
79
|
// 2. FFN
|
|
80
|
+
// Stage the residual tensor so that runDenseFFNGPU's ffn_down matmul can
|
|
81
|
+
// opportunistically fuse with it (WideTile+residual kernel). The flag
|
|
82
|
+
// `context.__ffnResidualFusedFired` will be set to true by dense.js if
|
|
83
|
+
// the fusion actually fired; we read it below to skip the separate add.
|
|
84
|
+
// If the pre-norm residual sum was captured (fused RMSNorm path), use that
|
|
85
|
+
// buffer directly — it IS the correct residual for ffn_residual because
|
|
86
|
+
// ffn_residual adds to the pre-rmsnorm tensor.
|
|
87
|
+
const pendingResidualForFfn = prenormSumBuffer
|
|
88
|
+
? { buffer: prenormSumBuffer, dtype: postAttn.dtype, shape: postAttn.shape }
|
|
89
|
+
: postAttn;
|
|
90
|
+
context.__pendingFfnResidualTensor = pendingResidualForFfn;
|
|
91
|
+
context.__ffnResidualFusedFired = false;
|
|
80
92
|
|
|
81
93
|
let ffnOutput;
|
|
82
94
|
if (config.useMoE && isMoELayerLocal(layerIdx, config, layerWeights)) {
|
|
@@ -84,6 +96,9 @@ export async function processFFNStandard(
|
|
|
84
96
|
} else {
|
|
85
97
|
ffnOutput = await runDenseFFNGPU(layerIdx, normedTensor, numTokens, context, layerWeights);
|
|
86
98
|
}
|
|
99
|
+
const ffnResidualFused = context.__ffnResidualFusedFired === true;
|
|
100
|
+
context.__pendingFfnResidualTensor = null;
|
|
101
|
+
context.__ffnResidualFusedFired = false;
|
|
87
102
|
await runProbes('ffn_out', ffnOutput.buffer, {
|
|
88
103
|
layerIdx,
|
|
89
104
|
numTokens,
|
|
@@ -96,20 +111,28 @@ export async function processFFNStandard(
|
|
|
96
111
|
await debugFFNBuffer(context, layerIdx, 'FFN output', ffnOutput, numTokens, hiddenSize);
|
|
97
112
|
|
|
98
113
|
// 3. Residual add (uses prenorm sum when fused, otherwise postAttn)
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
114
|
+
// When WideTile+residual fusion fired inside runDenseFFNGPU, ffnOutput
|
|
115
|
+
// IS already post-residual — skip the separate add entirely.
|
|
116
|
+
let output;
|
|
102
117
|
let residualInput = ffnOutput;
|
|
103
|
-
|
|
104
|
-
if (
|
|
105
|
-
|
|
118
|
+
let residualInputOwned = false;
|
|
119
|
+
if (ffnResidualFused) {
|
|
120
|
+
output = ffnOutput;
|
|
121
|
+
} else {
|
|
122
|
+
const residualTensor = prenormSumBuffer
|
|
123
|
+
? { buffer: prenormSumBuffer, dtype: postAttn.dtype, shape: postAttn.shape }
|
|
124
|
+
: postAttn;
|
|
125
|
+
residualInputOwned = ffnOutput.dtype !== residualTensor.dtype;
|
|
126
|
+
if (residualInputOwned) {
|
|
127
|
+
residualInput = await doCast(ffnOutput, residualTensor.dtype, recorder);
|
|
128
|
+
}
|
|
129
|
+
output = await doResidualAdd(residualInput, residualTensor, size, recorder, {
|
|
130
|
+
label: `L${layerIdx}.ffn_residual`,
|
|
131
|
+
layerIdx,
|
|
132
|
+
outputBuffer: decodeOutputBuffer,
|
|
133
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
134
|
+
});
|
|
106
135
|
}
|
|
107
|
-
const output = await doResidualAdd(residualInput, residualTensor, size, recorder, {
|
|
108
|
-
label: `L${layerIdx}.ffn_residual`,
|
|
109
|
-
layerIdx,
|
|
110
|
-
outputBuffer: decodeOutputBuffer,
|
|
111
|
-
executionPolicies: context.executionPolicies ?? null,
|
|
112
|
-
});
|
|
113
136
|
await runProbes('layer_out', output.buffer, {
|
|
114
137
|
layerIdx,
|
|
115
138
|
numTokens,
|
|
@@ -131,7 +154,11 @@ export async function processFFNStandard(
|
|
|
131
154
|
if (residualInputOwned) {
|
|
132
155
|
releaseOrTrack(recorder, residualInput.buffer, decodeBuffers);
|
|
133
156
|
}
|
|
134
|
-
|
|
157
|
+
// When fusion fired, `output === ffnOutput` and we must NOT release it
|
|
158
|
+
// here — it's our return value.
|
|
159
|
+
if (!ffnResidualFused) {
|
|
160
|
+
releaseOrTrack(recorder, ffnOutput.buffer, decodeBuffers);
|
|
161
|
+
}
|
|
135
162
|
|
|
136
163
|
return output;
|
|
137
164
|
}
|