doppler-gpu 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +19 -15
- package/package.json +77 -29
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +64 -6
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +524 -350
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +12 -2
- package/src/config/schema/doppler.schema.js +108 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +68 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +76 -0
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +107 -0
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +3 -21
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +6 -5
- package/src/gpu/kernels/index.js +3 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +12 -0
- package/src/inference/browser-harness-text-helpers.js +223 -14
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +264 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +22 -27
- package/src/inference/pipelines/text/embed.js +0 -39
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +4 -5
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +8 -3
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +223 -547
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +5 -2
- package/src/inference/pipelines/text/init.js +183 -724
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.js +2 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +120 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +223 -27
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +13 -3
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +11 -3
- package/src/storage/artifact-storage-context.js +231 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +21 -24
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +2 -0
- package/src/tooling/source-runtime-bundle.js +3 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
resolveWeightBufferMaterialization,
|
|
13
13
|
} from '../../../../gpu/weight-buffer.js';
|
|
14
14
|
import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
|
|
15
|
+
import { getRuntimeConfig } from '../../../../config/runtime.js';
|
|
15
16
|
import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
|
|
16
17
|
import {
|
|
17
18
|
runFusedFFN,
|
|
@@ -27,6 +28,7 @@ import { isKernelDebugEnabled, dumpTokenVector } from '../debug-utils.js';
|
|
|
27
28
|
import { applyLoRA } from '../lora-apply.js';
|
|
28
29
|
import { getLoRAModule } from '../lora.js';
|
|
29
30
|
import { getWeightBuffer, getNormWeightBuffer } from '../weights.js';
|
|
31
|
+
import { runProbes } from '../probes.js';
|
|
30
32
|
import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
31
33
|
import {
|
|
32
34
|
getKernelPathMatmulPrecision,
|
|
@@ -145,8 +147,20 @@ export function resolveDenseFFNFusedPathDtypes(options = {}) {
|
|
|
145
147
|
}
|
|
146
148
|
|
|
147
149
|
function hasExplicitMatmulPrecision(role, phase, layerIdx, kernelPath) {
|
|
150
|
+
// Only treat precision as "explicit split" when declared on the role's OWN
|
|
151
|
+
// step. The FUSED_FFN_PRECISION_FALLBACK_ROLES fallback that resolves via
|
|
152
|
+
// the aggregate `ffn` step's precision applies identically to gate/up/down
|
|
153
|
+
// and must NOT force the split path — otherwise a manifest-declared `ffn`
|
|
154
|
+
// precision permanently blocks the fused gate_up_activation kernel.
|
|
148
155
|
const precision = getKernelPathMatmulPrecision(role, phase, layerIdx, kernelPath);
|
|
149
|
-
|
|
156
|
+
if (!precision) return false;
|
|
157
|
+
const fallbackPrecision = getKernelPathMatmulPrecision('ffn', phase, layerIdx, kernelPath);
|
|
158
|
+
const inheritedFromFfn = fallbackPrecision
|
|
159
|
+
&& fallbackPrecision.inputDtype === precision.inputDtype
|
|
160
|
+
&& fallbackPrecision.outputDtype === precision.outputDtype
|
|
161
|
+
&& fallbackPrecision.activationDtype === precision.activationDtype;
|
|
162
|
+
if (inheritedFromFfn) return false;
|
|
163
|
+
return precision.inputDtype != null || precision.outputDtype != null;
|
|
150
164
|
}
|
|
151
165
|
|
|
152
166
|
export function canUseNativeF16FusedGateUp(options = {}) {
|
|
@@ -325,7 +339,7 @@ export async function runDenseFFNGPU(
|
|
|
325
339
|
if (!device) throw new Error('No GPU device');
|
|
326
340
|
|
|
327
341
|
const { config, recorder } = context;
|
|
328
|
-
const { hiddenSize, hiddenActivation, swigluLimit } = config;
|
|
342
|
+
const { hiddenSize, hiddenActivation, swigluLimit, useDoubleWideMlp } = config;
|
|
329
343
|
const intermediateSize = resolveLayerIntermediateSize(config, layerIdx);
|
|
330
344
|
const lastTokenIdx = Math.max(0, numTokens - 1);
|
|
331
345
|
const lora = context.lora || null;
|
|
@@ -434,6 +448,22 @@ export async function runDenseFFNGPU(
|
|
|
434
448
|
releaseBuffer(gateUpOutput.buffer);
|
|
435
449
|
}
|
|
436
450
|
|
|
451
|
+
// Opt-in WideTile+residual fusion: if the caller (processFFNStandard)
|
|
452
|
+
// staged a residual tensor on the context AND no LoRA is present on
|
|
453
|
+
// down_proj (LoRA requires the pre-residual output for its add), route
|
|
454
|
+
// this matmul to the q4_fused_widetile_residual variant which produces
|
|
455
|
+
// (ffn_down_result + residual) in one dispatch. Tell the caller via a
|
|
456
|
+
// context flag so processFFNStandard skips its downstream doResidualAdd.
|
|
457
|
+
const pendingResidual = context.__pendingFfnResidualTensor;
|
|
458
|
+
const downLoraProbe = getLoRAModule(lora, layerIdx, 'down_proj');
|
|
459
|
+
const mergedSession = getRuntimeConfig().inference?.session;
|
|
460
|
+
const tryFuseDownResidual = pendingResidual != null
|
|
461
|
+
&& !downLoraProbe
|
|
462
|
+
&& numTokens > 1
|
|
463
|
+
&& activatedOutput.dtype === 'f32'
|
|
464
|
+
&& downOutputDtype === 'f32'
|
|
465
|
+
&& pendingResidual.dtype === 'f32';
|
|
466
|
+
let residualFusedHere = false;
|
|
437
467
|
let output = await doMatmul(
|
|
438
468
|
activatedOutput, downWeight,
|
|
439
469
|
numTokens, hiddenSize, intermediateSize,
|
|
@@ -445,9 +475,26 @@ export async function runDenseFFNGPU(
|
|
|
445
475
|
outputDtype: downOutputDtype,
|
|
446
476
|
role: 'ffn_down',
|
|
447
477
|
executionPolicies: context.executionPolicies ?? null,
|
|
478
|
+
residualTensor: tryFuseDownResidual ? pendingResidual : null,
|
|
448
479
|
},
|
|
449
480
|
recorder
|
|
450
481
|
);
|
|
482
|
+
// Detect whether the fusion fired: if the selected variant for ffn_down
|
|
483
|
+
// matmul was q4_fused_widetile_residual, then output IS post-residual.
|
|
484
|
+
// We infer this cheaply by re-checking the conditions the selector uses.
|
|
485
|
+
// (A cleaner signal would require a return-shape change across all
|
|
486
|
+
// dense.js paths; this local signal is enough for correctness.)
|
|
487
|
+
{
|
|
488
|
+
if (tryFuseDownResidual
|
|
489
|
+
&& getKernelCapabilities().hasF16 === true
|
|
490
|
+
&& mergedSession?.useWideTileResidualFusion === true
|
|
491
|
+
&& mergedSession?.useWideTileQ4KPrefill === true
|
|
492
|
+
&& mergedSession?.retainQ4KMaterialization === true
|
|
493
|
+
) {
|
|
494
|
+
residualFusedHere = true;
|
|
495
|
+
context.__ffnResidualFusedFired = true;
|
|
496
|
+
}
|
|
497
|
+
}
|
|
451
498
|
|
|
452
499
|
const loraDown = getLoRAModule(lora, layerIdx, 'down_proj');
|
|
453
500
|
if (loraDown) {
|
|
@@ -537,6 +584,7 @@ export async function runDenseFFNGPU(
|
|
|
537
584
|
f16BatchSupported,
|
|
538
585
|
batchSize: numTokens,
|
|
539
586
|
hiddenSizeAligned32,
|
|
587
|
+
useDoubleWideMlp: Boolean(useDoubleWideMlp),
|
|
540
588
|
});
|
|
541
589
|
const useFusedGateUp = gateUpPathMode === 'split'
|
|
542
590
|
? false
|
|
@@ -608,6 +656,16 @@ export async function runDenseFFNGPU(
|
|
|
608
656
|
releaseOrTrack(recorder, isWeightBuffer(upWeight) ? upWeight.buffer : upWeight);
|
|
609
657
|
}
|
|
610
658
|
|
|
659
|
+
// Opt-in WideTile+residual fusion (fused-gate-up path).
|
|
660
|
+
const pendingResidualFused = context.__pendingFfnResidualTensor;
|
|
661
|
+
const downLoraProbeFused = getLoRAModule(lora, layerIdx, 'down_proj');
|
|
662
|
+
const mergedSessionFused = getRuntimeConfig().inference?.session;
|
|
663
|
+
const tryFuseDownResidualFused = pendingResidualFused != null
|
|
664
|
+
&& !downLoraProbeFused
|
|
665
|
+
&& numTokens > 1
|
|
666
|
+
&& downInput.dtype === 'f32'
|
|
667
|
+
&& fusedDownOutputDtype === 'f32'
|
|
668
|
+
&& pendingResidualFused.dtype === 'f32';
|
|
611
669
|
let output = await doMatmul(
|
|
612
670
|
downInput,
|
|
613
671
|
downWeight,
|
|
@@ -622,9 +680,20 @@ export async function runDenseFFNGPU(
|
|
|
622
680
|
outputDtype: fusedDownOutputDtype,
|
|
623
681
|
role: 'ffn_down',
|
|
624
682
|
executionPolicies: context.executionPolicies ?? null,
|
|
683
|
+
residualTensor: tryFuseDownResidualFused ? pendingResidualFused : null,
|
|
625
684
|
},
|
|
626
685
|
recorder
|
|
627
686
|
);
|
|
687
|
+
{
|
|
688
|
+
if (tryFuseDownResidualFused
|
|
689
|
+
&& getKernelCapabilities().hasF16 === true
|
|
690
|
+
&& mergedSessionFused?.useWideTileResidualFusion === true
|
|
691
|
+
&& mergedSessionFused?.useWideTileQ4KPrefill === true
|
|
692
|
+
&& mergedSessionFused?.retainQ4KMaterialization === true
|
|
693
|
+
) {
|
|
694
|
+
context.__ffnResidualFusedFired = true;
|
|
695
|
+
}
|
|
696
|
+
}
|
|
628
697
|
|
|
629
698
|
const loraDown = getLoRAModule(lora, layerIdx, 'down_proj');
|
|
630
699
|
if (loraDown) {
|
|
@@ -748,6 +817,84 @@ export async function runDenseFFNGPU(
|
|
|
748
817
|
});
|
|
749
818
|
sharedInputOwned = sharedInputTensor !== inputTensor;
|
|
750
819
|
}
|
|
820
|
+
// Opt-in fused gate + up + GeGLU path. Replaces 3 separate dispatches
|
|
821
|
+
// (gate_proj + up_proj + gelu activation) with a single fused kernel when
|
|
822
|
+
// preconditions match: prefill (M>1), f16-materialisable weights + f16
|
|
823
|
+
// activations, gelu activation, no LoRA on gate/up. Gated by
|
|
824
|
+
// runtime.inference.session.useFusedGateUpGelu (default false).
|
|
825
|
+
const gateF16 = resolveWeightBufferMaterialization(layerWeights.gate, 'f16');
|
|
826
|
+
const upF16 = resolveWeightBufferMaterialization(layerWeights.up, 'f16');
|
|
827
|
+
const gateF16Dtype = getWeightDtype(gateF16);
|
|
828
|
+
const upF16Dtype = getWeightDtype(upF16);
|
|
829
|
+
const earlyLoraGate = getLoRAModule(lora, layerIdx, 'gate_proj');
|
|
830
|
+
const earlyLoraUp = getLoRAModule(lora, layerIdx, 'up_proj');
|
|
831
|
+
const fusedGateUpGeluCandidate = context.useFusedGateUpGelu === true
|
|
832
|
+
&& numTokens > 1
|
|
833
|
+
&& hiddenActivation === 'gelu'
|
|
834
|
+
&& !earlyLoraGate && !earlyLoraUp
|
|
835
|
+
&& sharedInputTensor.dtype === 'f16'
|
|
836
|
+
&& gateF16Dtype === 'f16'
|
|
837
|
+
&& upF16Dtype === 'f16';
|
|
838
|
+
if (fusedGateUpGeluCandidate) {
|
|
839
|
+
const { runFusedGateUpGelu, recordFusedGateUpGelu } =
|
|
840
|
+
await import('../../../../gpu/kernels/fused-gate-up-gelu.js');
|
|
841
|
+
const fused = recorder
|
|
842
|
+
? await recordFusedGateUpGelu(recorder, sharedInputTensor, gateF16, upF16, {
|
|
843
|
+
M: numTokens,
|
|
844
|
+
hiddenSize,
|
|
845
|
+
intermediateSize,
|
|
846
|
+
transposeB: true,
|
|
847
|
+
})
|
|
848
|
+
: await runFusedGateUpGelu(sharedInputTensor, gateF16, upF16, {
|
|
849
|
+
M: numTokens,
|
|
850
|
+
hiddenSize,
|
|
851
|
+
intermediateSize,
|
|
852
|
+
transposeB: true,
|
|
853
|
+
});
|
|
854
|
+
if (sharedInputOwned) {
|
|
855
|
+
if (recorder) { recorder.trackTemporaryBuffer(sharedInputTensor.buffer); }
|
|
856
|
+
else { releaseBuffer(sharedInputTensor.buffer); }
|
|
857
|
+
}
|
|
858
|
+
// Proceed directly to down_proj with the fused activation output.
|
|
859
|
+
const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
|
|
860
|
+
let downInputTensor = fused;
|
|
861
|
+
let downInputOwned = false;
|
|
862
|
+
if (downInputDtype && fused.dtype !== downInputDtype) {
|
|
863
|
+
downInputTensor = await coerceTensorDtype(fused, downInputDtype, recorder, {
|
|
864
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
865
|
+
op: 'ffn_down_input',
|
|
866
|
+
transitionDeclaredBy: 'step_precision',
|
|
867
|
+
});
|
|
868
|
+
downInputOwned = downInputTensor !== fused;
|
|
869
|
+
}
|
|
870
|
+
let outFused = await doMatmul(
|
|
871
|
+
downInputTensor,
|
|
872
|
+
downWeight,
|
|
873
|
+
numTokens,
|
|
874
|
+
hiddenSize,
|
|
875
|
+
intermediateSize,
|
|
876
|
+
{
|
|
877
|
+
transposeB: 'auto',
|
|
878
|
+
label: `L${layerIdx}.ffn_down`,
|
|
879
|
+
layerIdx,
|
|
880
|
+
kernelPath,
|
|
881
|
+
outputDtype: downOutputDtype,
|
|
882
|
+
role: 'ffn_down',
|
|
883
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
884
|
+
},
|
|
885
|
+
recorder
|
|
886
|
+
);
|
|
887
|
+
if (!isGpuBufferInstance(layerWeights.down) && !isWeightBuffer(layerWeights.down)) {
|
|
888
|
+
releaseOrTrack(recorder, isWeightBuffer(downWeight) ? downWeight.buffer : downWeight);
|
|
889
|
+
}
|
|
890
|
+
if (downInputOwned) {
|
|
891
|
+
if (recorder) { recorder.trackTemporaryBuffer(downInputTensor.buffer); }
|
|
892
|
+
else { releaseBuffer(downInputTensor.buffer); }
|
|
893
|
+
}
|
|
894
|
+
if (recorder) { recorder.trackTemporaryBuffer(fused.buffer); }
|
|
895
|
+
else { releaseBuffer(fused.buffer); }
|
|
896
|
+
return outFused;
|
|
897
|
+
}
|
|
751
898
|
const gateWeight = getWeightBuffer(layerWeights.gate, 'ffn_gate');
|
|
752
899
|
let gateOutput = await doMatmul(
|
|
753
900
|
gateInputDtype === sharedInputTensor.dtype ? sharedInputTensor : inputTensor,
|
|
@@ -849,6 +996,25 @@ export async function runDenseFFNGPU(
|
|
|
849
996
|
});
|
|
850
997
|
}
|
|
851
998
|
|
|
999
|
+
await runProbes('ffn_gate', gateOutput.buffer, {
|
|
1000
|
+
layerIdx,
|
|
1001
|
+
numTokens,
|
|
1002
|
+
hiddenSize: intermediateSize,
|
|
1003
|
+
probes: context.debugProbes,
|
|
1004
|
+
recorder,
|
|
1005
|
+
operatorDiagnostics: context.operatorDiagnostics,
|
|
1006
|
+
dtype: gateOutput.dtype,
|
|
1007
|
+
});
|
|
1008
|
+
await runProbes('ffn_up', upOutput.buffer, {
|
|
1009
|
+
layerIdx,
|
|
1010
|
+
numTokens,
|
|
1011
|
+
hiddenSize: intermediateSize,
|
|
1012
|
+
probes: context.debugProbes,
|
|
1013
|
+
recorder,
|
|
1014
|
+
operatorDiagnostics: context.operatorDiagnostics,
|
|
1015
|
+
dtype: upOutput.dtype,
|
|
1016
|
+
});
|
|
1017
|
+
|
|
852
1018
|
const activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
|
|
853
1019
|
size: numTokens * intermediateSize,
|
|
854
1020
|
gate: gateOutput,
|
|
@@ -866,6 +1032,16 @@ export async function runDenseFFNGPU(
|
|
|
866
1032
|
});
|
|
867
1033
|
}
|
|
868
1034
|
|
|
1035
|
+
await runProbes('ffn_act', activatedOutput.buffer, {
|
|
1036
|
+
layerIdx,
|
|
1037
|
+
numTokens,
|
|
1038
|
+
hiddenSize: intermediateSize,
|
|
1039
|
+
probes: context.debugProbes,
|
|
1040
|
+
recorder,
|
|
1041
|
+
operatorDiagnostics: context.operatorDiagnostics,
|
|
1042
|
+
dtype: activatedOutput.dtype,
|
|
1043
|
+
});
|
|
1044
|
+
|
|
869
1045
|
if (recorder) {
|
|
870
1046
|
recorder.trackTemporaryBuffer(gateOutput.buffer);
|
|
871
1047
|
recorder.trackTemporaryBuffer(upOutput.buffer);
|
|
@@ -975,7 +1151,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
|
|
|
975
1151
|
if (!device) throw new Error('No GPU device');
|
|
976
1152
|
|
|
977
1153
|
const { config, weightConfig, debugFlags, recorder } = context;
|
|
978
|
-
const { hiddenSize, hiddenActivation, swigluLimit } = config;
|
|
1154
|
+
const { hiddenSize, hiddenActivation, swigluLimit, useDoubleWideMlp } = config;
|
|
979
1155
|
const intermediateSize = resolveLayerIntermediateSize(config, layerIdx);
|
|
980
1156
|
const lora = context.lora || null;
|
|
981
1157
|
const ffnStepPrecision = context.ffnStepPrecision ?? null;
|
|
@@ -1124,6 +1300,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
|
|
|
1124
1300
|
f16BatchSupported: getKernelCapabilities().hasF16,
|
|
1125
1301
|
batchSize: numTokens,
|
|
1126
1302
|
hiddenSizeAligned32,
|
|
1303
|
+
useDoubleWideMlp: Boolean(useDoubleWideMlp),
|
|
1127
1304
|
});
|
|
1128
1305
|
const canUseFusedGateUp = gateUpPathMode === 'split'
|
|
1129
1306
|
? false
|
|
@@ -77,6 +77,18 @@ export async function processFFNStandard(
|
|
|
77
77
|
await debugFFNBuffer(context, layerIdx, 'FFN input', normedTensor, numTokens, hiddenSize);
|
|
78
78
|
|
|
79
79
|
// 2. FFN
|
|
80
|
+
// Stage the residual tensor so that runDenseFFNGPU's ffn_down matmul can
|
|
81
|
+
// opportunistically fuse with it (WideTile+residual kernel). The flag
|
|
82
|
+
// `context.__ffnResidualFusedFired` will be set to true by dense.js if
|
|
83
|
+
// the fusion actually fired; we read it below to skip the separate add.
|
|
84
|
+
// If the pre-norm residual sum was captured (fused RMSNorm path), use that
|
|
85
|
+
// buffer directly — it IS the correct residual for ffn_residual because
|
|
86
|
+
// ffn_residual adds to the pre-rmsnorm tensor.
|
|
87
|
+
const pendingResidualForFfn = prenormSumBuffer
|
|
88
|
+
? { buffer: prenormSumBuffer, dtype: postAttn.dtype, shape: postAttn.shape }
|
|
89
|
+
: postAttn;
|
|
90
|
+
context.__pendingFfnResidualTensor = pendingResidualForFfn;
|
|
91
|
+
context.__ffnResidualFusedFired = false;
|
|
80
92
|
|
|
81
93
|
let ffnOutput;
|
|
82
94
|
if (config.useMoE && isMoELayerLocal(layerIdx, config, layerWeights)) {
|
|
@@ -84,6 +96,9 @@ export async function processFFNStandard(
|
|
|
84
96
|
} else {
|
|
85
97
|
ffnOutput = await runDenseFFNGPU(layerIdx, normedTensor, numTokens, context, layerWeights);
|
|
86
98
|
}
|
|
99
|
+
const ffnResidualFused = context.__ffnResidualFusedFired === true;
|
|
100
|
+
context.__pendingFfnResidualTensor = null;
|
|
101
|
+
context.__ffnResidualFusedFired = false;
|
|
87
102
|
await runProbes('ffn_out', ffnOutput.buffer, {
|
|
88
103
|
layerIdx,
|
|
89
104
|
numTokens,
|
|
@@ -96,20 +111,28 @@ export async function processFFNStandard(
|
|
|
96
111
|
await debugFFNBuffer(context, layerIdx, 'FFN output', ffnOutput, numTokens, hiddenSize);
|
|
97
112
|
|
|
98
113
|
// 3. Residual add (uses prenorm sum when fused, otherwise postAttn)
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
114
|
+
// When WideTile+residual fusion fired inside runDenseFFNGPU, ffnOutput
|
|
115
|
+
// IS already post-residual — skip the separate add entirely.
|
|
116
|
+
let output;
|
|
102
117
|
let residualInput = ffnOutput;
|
|
103
|
-
|
|
104
|
-
if (
|
|
105
|
-
|
|
118
|
+
let residualInputOwned = false;
|
|
119
|
+
if (ffnResidualFused) {
|
|
120
|
+
output = ffnOutput;
|
|
121
|
+
} else {
|
|
122
|
+
const residualTensor = prenormSumBuffer
|
|
123
|
+
? { buffer: prenormSumBuffer, dtype: postAttn.dtype, shape: postAttn.shape }
|
|
124
|
+
: postAttn;
|
|
125
|
+
residualInputOwned = ffnOutput.dtype !== residualTensor.dtype;
|
|
126
|
+
if (residualInputOwned) {
|
|
127
|
+
residualInput = await doCast(ffnOutput, residualTensor.dtype, recorder);
|
|
128
|
+
}
|
|
129
|
+
output = await doResidualAdd(residualInput, residualTensor, size, recorder, {
|
|
130
|
+
label: `L${layerIdx}.ffn_residual`,
|
|
131
|
+
layerIdx,
|
|
132
|
+
outputBuffer: decodeOutputBuffer,
|
|
133
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
134
|
+
});
|
|
106
135
|
}
|
|
107
|
-
const output = await doResidualAdd(residualInput, residualTensor, size, recorder, {
|
|
108
|
-
label: `L${layerIdx}.ffn_residual`,
|
|
109
|
-
layerIdx,
|
|
110
|
-
outputBuffer: decodeOutputBuffer,
|
|
111
|
-
executionPolicies: context.executionPolicies ?? null,
|
|
112
|
-
});
|
|
113
136
|
await runProbes('layer_out', output.buffer, {
|
|
114
137
|
layerIdx,
|
|
115
138
|
numTokens,
|
|
@@ -131,7 +154,11 @@ export async function processFFNStandard(
|
|
|
131
154
|
if (residualInputOwned) {
|
|
132
155
|
releaseOrTrack(recorder, residualInput.buffer, decodeBuffers);
|
|
133
156
|
}
|
|
134
|
-
|
|
157
|
+
// When fusion fired, `output === ffnOutput` and we must NOT release it
|
|
158
|
+
// here — it's our return value.
|
|
159
|
+
if (!ffnResidualFused) {
|
|
160
|
+
releaseOrTrack(recorder, ffnOutput.buffer, decodeBuffers);
|
|
161
|
+
}
|
|
135
162
|
|
|
136
163
|
return output;
|
|
137
164
|
}
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
|
|
2
|
+
import { log } from '../../../debug/index.js';
|
|
3
|
+
import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
|
|
4
|
+
import { resolveActiveExecutionPlan } from './execution-plan.js';
|
|
5
|
+
import { resolvePerLayerInputsSession } from './generator-helpers.js';
|
|
6
|
+
import {
|
|
7
|
+
ensurePleGpuHotVocabularyRuntime,
|
|
8
|
+
ensurePleGpuSplitTablesRuntime,
|
|
9
|
+
ensurePleScaledProjectionNormWeight,
|
|
10
|
+
} from './per-layer-inputs.js';
|
|
11
|
+
|
|
12
|
+
export function shouldDisableBatchDecodeAfterShortBatch({ hitStop, actualCount, requestedCount }) {
|
|
13
|
+
return hitStop !== true
|
|
14
|
+
&& Number.isInteger(actualCount)
|
|
15
|
+
&& Number.isInteger(requestedCount)
|
|
16
|
+
&& actualCount > 0
|
|
17
|
+
&& actualCount < requestedCount;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function resolveHotVocabularyBatchDecodeAvailability({
|
|
21
|
+
hasRangeBackedPerLayerInputs,
|
|
22
|
+
pleHotVocabularyRuntime,
|
|
23
|
+
tokenId,
|
|
24
|
+
}) {
|
|
25
|
+
if (hasRangeBackedPerLayerInputs !== true) {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
if (!pleHotVocabularyRuntime || typeof pleHotVocabularyRuntime !== 'object') {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
const sentinelIndex = pleHotVocabularyRuntime.sentinelIndex;
|
|
32
|
+
if (!Number.isInteger(sentinelIndex)) {
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
if (!Number.isInteger(tokenId)) {
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
const hotTokenIndex = pleHotVocabularyRuntime.hotTokenIndexMap?.[tokenId] ?? sentinelIndex;
|
|
39
|
+
return hotTokenIndex !== sentinelIndex;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
|
|
43
|
+
const perLayerInputsSession = resolvePerLayerInputsSession(
|
|
44
|
+
state.modelConfig?.perLayerInputsSession ?? null,
|
|
45
|
+
state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
46
|
+
);
|
|
47
|
+
const materialization = perLayerInputsSession?.materialization;
|
|
48
|
+
if (state.debug) {
|
|
49
|
+
log.debug(
|
|
50
|
+
'Pipeline',
|
|
51
|
+
`PLE materialization session=${materialization ?? 'null'} ` +
|
|
52
|
+
`modelId=${state.modelConfig?.modelId ?? 'unknown'} ` +
|
|
53
|
+
`runtimePerLayerInputs=${Boolean(state.runtimeConfig.inference.session?.perLayerInputs)} ` +
|
|
54
|
+
`manifestPerLayerInputs=${Boolean(state.modelConfig?.perLayerInputsSession)}`
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
await ensurePleGpuHotVocabularyRuntime({
|
|
58
|
+
config: state.modelConfig,
|
|
59
|
+
weights: state.weights,
|
|
60
|
+
perLayerInputsSession,
|
|
61
|
+
debugFlags: state.debugFlags,
|
|
62
|
+
tokenizer: state.tokenizer ?? null,
|
|
63
|
+
seedTokenIds: Array.isArray(seedTokenIds) ? seedTokenIds : null,
|
|
64
|
+
});
|
|
65
|
+
await ensurePleGpuSplitTablesRuntime({
|
|
66
|
+
config: state.modelConfig,
|
|
67
|
+
weights: state.weights,
|
|
68
|
+
perLayerInputsSession,
|
|
69
|
+
debugFlags: state.debugFlags,
|
|
70
|
+
});
|
|
71
|
+
await ensurePleScaledProjectionNormWeight({
|
|
72
|
+
config: state.modelConfig,
|
|
73
|
+
weights: state.weights,
|
|
74
|
+
weightConfig: {
|
|
75
|
+
rmsNormWeightOffset: state.modelConfig?.rmsNormWeightOffset ?? false,
|
|
76
|
+
},
|
|
77
|
+
debugFlags: state.debugFlags,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function recordPrefillProfileStep(state, entry) {
|
|
82
|
+
if (!entry?.timings || Object.keys(entry.timings).length === 0) return;
|
|
83
|
+
if (!state.stats.prefillProfileSteps) {
|
|
84
|
+
state.stats.prefillProfileSteps = [];
|
|
85
|
+
}
|
|
86
|
+
state.stats.prefillProfileSteps.push(entry);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Resolve display text for a token ID sequence.
|
|
91
|
+
*
|
|
92
|
+
* Preference order:
|
|
93
|
+
* 1. renderTokenText (custom primary renderer) — if provided and returns non-empty string.
|
|
94
|
+
* 2. tokenizer.decode (default primary) — if renderTokenText is not provided.
|
|
95
|
+
* 3. renderFallbackTokenText (custom fallback) — if provided and returns non-empty string,
|
|
96
|
+
* unless primary returned empty string and fallback looks like a special token (<...>),
|
|
97
|
+
* in which case empty string is preserved to keep skip-special behavior deterministic.
|
|
98
|
+
* 4. tokenizer.decode with skipSpecialTokens=false (default fallback) — same special-token guard.
|
|
99
|
+
* 5. fallbackText — static fallback string (default '?').
|
|
100
|
+
*/
|
|
101
|
+
export function resolveTokenText(tokenizer, tokenIds, fallbackText = '?', renderTokenText, renderFallbackTokenText) {
|
|
102
|
+
const renderPrimary = typeof renderTokenText === 'function'
|
|
103
|
+
? renderTokenText
|
|
104
|
+
: (ids) => tokenizer?.decode?.(ids);
|
|
105
|
+
const renderFallback = typeof renderFallbackTokenText === 'function'
|
|
106
|
+
? renderFallbackTokenText
|
|
107
|
+
: (ids) => tokenizer?.decode?.(ids, false);
|
|
108
|
+
|
|
109
|
+
const primaryText = renderPrimary(tokenIds);
|
|
110
|
+
if (typeof primaryText === 'string' && primaryText.length > 0) {
|
|
111
|
+
return primaryText;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const fallback = renderFallback(tokenIds);
|
|
115
|
+
if (typeof fallback === 'string' && fallback.length > 0) {
|
|
116
|
+
// Keep skip-special behavior deterministic: if primary decoding filtered this
|
|
117
|
+
// token to empty, do not reintroduce obvious special-token text via fallback.
|
|
118
|
+
if (
|
|
119
|
+
primaryText === ''
|
|
120
|
+
&& /^<[^>\n]{1,80}>$/.test(fallback.trim())
|
|
121
|
+
) {
|
|
122
|
+
return '';
|
|
123
|
+
}
|
|
124
|
+
return fallback;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
return fallbackText;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export function usesReplayPrefillDecode(state) {
|
|
131
|
+
return state?.modelConfig?.decodeStrategy === 'replay_prefill';
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export function assertIncrementalDecodeSupport(state, operation) {
|
|
135
|
+
if (!usesReplayPrefillDecode(state)) {
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
throw new Error(
|
|
139
|
+
`[Pipeline] ${operation} is not supported for models that require replay-prefill decode. ` +
|
|
140
|
+
'Incremental KV-cache decode and prefix snapshots stay disabled when the model config does not resolve ' +
|
|
141
|
+
'explicit layerTypes for mixed-geometry/shared-KV decode.'
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export function summarizeExecutionPlan(plan) {
|
|
146
|
+
if (!plan) {
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
if (typeof plan !== 'object') {
|
|
150
|
+
log.warn('Pipeline', `summarizeExecutionPlan: expected object, got ${typeof plan}`);
|
|
151
|
+
return null;
|
|
152
|
+
}
|
|
153
|
+
if (typeof plan.id !== 'string') {
|
|
154
|
+
log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "id"');
|
|
155
|
+
}
|
|
156
|
+
if (typeof plan.activationDtype !== 'string') {
|
|
157
|
+
log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "activationDtype"');
|
|
158
|
+
}
|
|
159
|
+
return {
|
|
160
|
+
id: plan.id,
|
|
161
|
+
kernelPathId: plan.kernelPathId ?? null,
|
|
162
|
+
kernelPathSource: plan.kernelPathSource ?? 'none',
|
|
163
|
+
activationDtype: plan.activationDtype,
|
|
164
|
+
readbackInterval: plan.readbackInterval ?? null,
|
|
165
|
+
batchSize: plan.defaultBatchSize,
|
|
166
|
+
stopCheckMode: plan.defaultStopCheckMode,
|
|
167
|
+
ringTokens: plan.ringTokens ?? null,
|
|
168
|
+
ringStop: plan.ringStop ?? null,
|
|
169
|
+
ringStaging: plan.ringStaging ?? null,
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export function shouldRetryWithFinitenessFallback(error) {
|
|
174
|
+
if (error?.name === 'FinitenessError') {
|
|
175
|
+
return true;
|
|
176
|
+
}
|
|
177
|
+
const message = typeof error?.message === 'string'
|
|
178
|
+
? error.message
|
|
179
|
+
: (typeof error === 'string' ? error : '');
|
|
180
|
+
if (!message.startsWith('[Sampling]')) {
|
|
181
|
+
return false;
|
|
182
|
+
}
|
|
183
|
+
return message.includes('no finite candidate logits after masking the pad token')
|
|
184
|
+
|| message.includes('Softmax produced no finite candidate probabilities');
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export function createUnhandledFinitenessPolicyError(state, contextLabel, error) {
|
|
188
|
+
const activePlan = resolveActiveExecutionPlan(state);
|
|
189
|
+
const wrapped = new Error(
|
|
190
|
+
`[Pipeline] ${contextLabel}: finiteness guard triggered for kernelPath ` +
|
|
191
|
+
`"${activePlan.kernelPathId ?? 'none'}" under fail-fast policy. ` +
|
|
192
|
+
'Resolve the unstable path with an explicit capability-aware execution override, ' +
|
|
193
|
+
'or opt into alternate-plan recovery with ' +
|
|
194
|
+
'runtime.inference.compute.rangeAwareSelectiveWidening.onTrigger="fallback-plan".',
|
|
195
|
+
error instanceof Error ? { cause: error } : undefined
|
|
196
|
+
);
|
|
197
|
+
wrapped.name = error?.name === 'FinitenessError' ? error.name : 'FinitenessError';
|
|
198
|
+
return wrapped;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function assertResolvedKVDtype(kvDtype, contextLabel) {
|
|
202
|
+
if (kvDtype === 'f16' || kvDtype === 'f32') {
|
|
203
|
+
return kvDtype;
|
|
204
|
+
}
|
|
205
|
+
throw new Error(
|
|
206
|
+
`[Pipeline] ${contextLabel}: expected execution-plan kvDtype to resolve to "f16" or "f32", ` +
|
|
207
|
+
`got ${JSON.stringify(kvDtype)}.`
|
|
208
|
+
);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export function resolveTargetPlanKVDtype(plan, contextLabel) {
|
|
212
|
+
return assertResolvedKVDtype(
|
|
213
|
+
plan?.kernelPath?.kvDtype ?? plan?.activationDtype ?? null,
|
|
214
|
+
contextLabel
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
export function resolveCurrentKVCacheDtype(state, plan, contextLabel) {
|
|
219
|
+
return assertResolvedKVDtype(
|
|
220
|
+
state?.kvCache?.kvDtype
|
|
221
|
+
?? plan?.kernelPath?.kvDtype
|
|
222
|
+
?? state?.runtimeConfig?.inference?.session?.kvcache?.kvDtype
|
|
223
|
+
?? plan?.activationDtype
|
|
224
|
+
?? null,
|
|
225
|
+
contextLabel
|
|
226
|
+
);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export function cloneRuntimeInferenceWithKVDtype(state, kvDtype) {
|
|
230
|
+
const runtimeInference = state?.runtimeConfig?.inference;
|
|
231
|
+
if (!runtimeInference?.session?.kvcache) {
|
|
232
|
+
throw new Error(
|
|
233
|
+
'[Pipeline] runtime.inference.session.kvcache is required for finiteness fallback KV-cache recovery.'
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
return mergeRuntimeValues(runtimeInference, {
|
|
237
|
+
session: {
|
|
238
|
+
kvcache: {
|
|
239
|
+
kvDtype,
|
|
240
|
+
},
|
|
241
|
+
},
|
|
242
|
+
});
|
|
243
|
+
}
|
|
@@ -40,8 +40,6 @@ export declare function releaseSharedAttentionState(
|
|
|
40
40
|
} | null
|
|
41
41
|
): void;
|
|
42
42
|
|
|
43
|
-
export declare function getWeightBufferConfig(state: PipelineState): WeightBufferConfig;
|
|
44
|
-
|
|
45
43
|
export declare function getLogitsWeights(state: PipelineState): LogitsWeights;
|
|
46
44
|
|
|
47
45
|
export declare function getLogitsConfig(state: PipelineState): LogitsConfig;
|
|
@@ -8,10 +8,7 @@ import { decodeReadback } from './debug-utils/index.js';
|
|
|
8
8
|
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
|
|
9
9
|
import { resolveRangeAwareSelectiveWideningConfig } from './finiteness-policy.js';
|
|
10
10
|
import { resolveActiveExecutionPlan } from './execution-plan.js';
|
|
11
|
-
|
|
12
|
-
function isPlainObject(value) {
|
|
13
|
-
return value != null && typeof value === 'object' && !Array.isArray(value);
|
|
14
|
-
}
|
|
11
|
+
import { isPlainObject } from '../../../utils/plain-object.js';
|
|
15
12
|
|
|
16
13
|
export function resolvePerLayerInputsSession(manifestSession, runtimeSession) {
|
|
17
14
|
if (!isPlainObject(runtimeSession)) {
|
|
@@ -161,6 +158,7 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
|
|
|
161
158
|
step: state.decodeStepCount,
|
|
162
159
|
phase: isDecodeMode ? 'decode' : 'prefill',
|
|
163
160
|
operatorDiagnostics: state.operatorDiagnostics,
|
|
161
|
+
useFusedGateUpGelu: state.runtimeConfig?.inference?.session?.useFusedGateUpGelu === true,
|
|
164
162
|
};
|
|
165
163
|
}
|
|
166
164
|
|
|
@@ -191,7 +189,7 @@ export function releaseSharedAttentionState(sharedAttentionState, recorder = nul
|
|
|
191
189
|
}
|
|
192
190
|
|
|
193
191
|
|
|
194
|
-
|
|
192
|
+
function getWeightBufferConfig(state) {
|
|
195
193
|
return {
|
|
196
194
|
rmsNormWeightOffset: state.modelConfig.rmsNormWeightOffset,
|
|
197
195
|
};
|
|
@@ -228,5 +226,6 @@ export function getLogitsConfig(state) {
|
|
|
228
226
|
activationDtype: effectiveActivationDtype,
|
|
229
227
|
kernelPath: activeKernelPath,
|
|
230
228
|
executionPolicies: state.executionV1State?.policies ?? null,
|
|
229
|
+
debugProbes: state.runtimeConfig.shared.debug.probes,
|
|
231
230
|
};
|
|
232
231
|
}
|