doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
|
|
2
|
+
import { log } from '../../../debug/index.js';
|
|
3
|
+
import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
|
|
4
|
+
import { resolveActiveExecutionPlan } from './execution-plan.js';
|
|
5
|
+
import { resolvePerLayerInputsSession } from './generator-helpers.js';
|
|
6
|
+
import {
|
|
7
|
+
ensurePleGpuHotVocabularyRuntime,
|
|
8
|
+
ensurePleGpuSplitTablesRuntime,
|
|
9
|
+
ensurePleScaledProjectionNormWeight,
|
|
10
|
+
} from './per-layer-inputs.js';
|
|
11
|
+
|
|
12
|
+
export function shouldDisableBatchDecodeAfterShortBatch({ hitStop, actualCount, requestedCount }) {
|
|
13
|
+
return hitStop !== true
|
|
14
|
+
&& Number.isInteger(actualCount)
|
|
15
|
+
&& Number.isInteger(requestedCount)
|
|
16
|
+
&& actualCount > 0
|
|
17
|
+
&& actualCount < requestedCount;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function resolveHotVocabularyBatchDecodeAvailability({
|
|
21
|
+
hasRangeBackedPerLayerInputs,
|
|
22
|
+
pleHotVocabularyRuntime,
|
|
23
|
+
tokenId,
|
|
24
|
+
}) {
|
|
25
|
+
if (hasRangeBackedPerLayerInputs !== true) {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
if (!pleHotVocabularyRuntime || typeof pleHotVocabularyRuntime !== 'object') {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
const sentinelIndex = pleHotVocabularyRuntime.sentinelIndex;
|
|
32
|
+
if (!Number.isInteger(sentinelIndex)) {
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
if (!Number.isInteger(tokenId)) {
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
const hotTokenIndex = pleHotVocabularyRuntime.hotTokenIndexMap?.[tokenId] ?? sentinelIndex;
|
|
39
|
+
return hotTokenIndex !== sentinelIndex;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
|
|
43
|
+
const perLayerInputsSession = resolvePerLayerInputsSession(
|
|
44
|
+
state.modelConfig?.perLayerInputsSession ?? null,
|
|
45
|
+
state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
46
|
+
);
|
|
47
|
+
const materialization = perLayerInputsSession?.materialization;
|
|
48
|
+
if (state.debug) {
|
|
49
|
+
log.debug(
|
|
50
|
+
'Pipeline',
|
|
51
|
+
`PLE materialization session=${materialization ?? 'null'} ` +
|
|
52
|
+
`modelId=${state.modelConfig?.modelId ?? 'unknown'} ` +
|
|
53
|
+
`runtimePerLayerInputs=${Boolean(state.runtimeConfig.inference.session?.perLayerInputs)} ` +
|
|
54
|
+
`manifestPerLayerInputs=${Boolean(state.modelConfig?.perLayerInputsSession)}`
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
await ensurePleGpuHotVocabularyRuntime({
|
|
58
|
+
config: state.modelConfig,
|
|
59
|
+
weights: state.weights,
|
|
60
|
+
perLayerInputsSession,
|
|
61
|
+
debugFlags: state.debugFlags,
|
|
62
|
+
tokenizer: state.tokenizer ?? null,
|
|
63
|
+
seedTokenIds: Array.isArray(seedTokenIds) ? seedTokenIds : null,
|
|
64
|
+
});
|
|
65
|
+
await ensurePleGpuSplitTablesRuntime({
|
|
66
|
+
config: state.modelConfig,
|
|
67
|
+
weights: state.weights,
|
|
68
|
+
perLayerInputsSession,
|
|
69
|
+
debugFlags: state.debugFlags,
|
|
70
|
+
});
|
|
71
|
+
await ensurePleScaledProjectionNormWeight({
|
|
72
|
+
config: state.modelConfig,
|
|
73
|
+
weights: state.weights,
|
|
74
|
+
weightConfig: {
|
|
75
|
+
rmsNormWeightOffset: state.modelConfig?.rmsNormWeightOffset ?? false,
|
|
76
|
+
},
|
|
77
|
+
debugFlags: state.debugFlags,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function recordPrefillProfileStep(state, entry) {
|
|
82
|
+
if (!entry?.timings || Object.keys(entry.timings).length === 0) return;
|
|
83
|
+
if (!state.stats.prefillProfileSteps) {
|
|
84
|
+
state.stats.prefillProfileSteps = [];
|
|
85
|
+
}
|
|
86
|
+
state.stats.prefillProfileSteps.push(entry);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Resolve display text for a token ID sequence.
|
|
91
|
+
*
|
|
92
|
+
* Preference order:
|
|
93
|
+
* 1. renderTokenText (custom primary renderer) — if provided and returns non-empty string.
|
|
94
|
+
* 2. tokenizer.decode (default primary) — if renderTokenText is not provided.
|
|
95
|
+
* 3. renderFallbackTokenText (custom fallback) — if provided and returns non-empty string,
|
|
96
|
+
* unless primary returned empty string and fallback looks like a special token (<...>),
|
|
97
|
+
* in which case empty string is preserved to keep skip-special behavior deterministic.
|
|
98
|
+
* 4. tokenizer.decode with skipSpecialTokens=false (default fallback) — same special-token guard.
|
|
99
|
+
* 5. fallbackText — static fallback string (default '?').
|
|
100
|
+
*/
|
|
101
|
+
export function resolveTokenText(tokenizer, tokenIds, fallbackText = '?', renderTokenText, renderFallbackTokenText) {
|
|
102
|
+
const renderPrimary = typeof renderTokenText === 'function'
|
|
103
|
+
? renderTokenText
|
|
104
|
+
: (ids) => tokenizer?.decode?.(ids);
|
|
105
|
+
const renderFallback = typeof renderFallbackTokenText === 'function'
|
|
106
|
+
? renderFallbackTokenText
|
|
107
|
+
: (ids) => tokenizer?.decode?.(ids, false);
|
|
108
|
+
|
|
109
|
+
const primaryText = renderPrimary(tokenIds);
|
|
110
|
+
if (typeof primaryText === 'string' && primaryText.length > 0) {
|
|
111
|
+
return primaryText;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const fallback = renderFallback(tokenIds);
|
|
115
|
+
if (typeof fallback === 'string' && fallback.length > 0) {
|
|
116
|
+
// Keep skip-special behavior deterministic: if primary decoding filtered this
|
|
117
|
+
// token to empty, do not reintroduce obvious special-token text via fallback.
|
|
118
|
+
if (
|
|
119
|
+
primaryText === ''
|
|
120
|
+
&& /^<[^>\n]{1,80}>$/.test(fallback.trim())
|
|
121
|
+
) {
|
|
122
|
+
return '';
|
|
123
|
+
}
|
|
124
|
+
return fallback;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
return fallbackText;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export function usesReplayPrefillDecode(state) {
|
|
131
|
+
return state?.modelConfig?.decodeStrategy === 'replay_prefill';
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export function assertIncrementalDecodeSupport(state, operation) {
|
|
135
|
+
if (!usesReplayPrefillDecode(state)) {
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
throw new Error(
|
|
139
|
+
`[Pipeline] ${operation} is not supported for models that require replay-prefill decode. ` +
|
|
140
|
+
'Incremental KV-cache decode and prefix snapshots stay disabled when the model config does not resolve ' +
|
|
141
|
+
'explicit layerTypes for mixed-geometry/shared-KV decode.'
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export function summarizeExecutionPlan(plan) {
|
|
146
|
+
if (!plan) {
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
if (typeof plan !== 'object') {
|
|
150
|
+
log.warn('Pipeline', `summarizeExecutionPlan: expected object, got ${typeof plan}`);
|
|
151
|
+
return null;
|
|
152
|
+
}
|
|
153
|
+
if (typeof plan.id !== 'string') {
|
|
154
|
+
log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "id"');
|
|
155
|
+
}
|
|
156
|
+
if (typeof plan.activationDtype !== 'string') {
|
|
157
|
+
log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "activationDtype"');
|
|
158
|
+
}
|
|
159
|
+
return {
|
|
160
|
+
id: plan.id,
|
|
161
|
+
kernelPathId: plan.kernelPathId ?? null,
|
|
162
|
+
kernelPathSource: plan.kernelPathSource ?? 'none',
|
|
163
|
+
activationDtype: plan.activationDtype,
|
|
164
|
+
readbackInterval: plan.readbackInterval ?? null,
|
|
165
|
+
batchSize: plan.defaultBatchSize,
|
|
166
|
+
stopCheckMode: plan.defaultStopCheckMode,
|
|
167
|
+
ringTokens: plan.ringTokens ?? null,
|
|
168
|
+
ringStop: plan.ringStop ?? null,
|
|
169
|
+
ringStaging: plan.ringStaging ?? null,
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export function shouldRetryWithFinitenessFallback(error) {
|
|
174
|
+
if (error?.name === 'FinitenessError') {
|
|
175
|
+
return true;
|
|
176
|
+
}
|
|
177
|
+
const message = typeof error?.message === 'string'
|
|
178
|
+
? error.message
|
|
179
|
+
: (typeof error === 'string' ? error : '');
|
|
180
|
+
if (!message.startsWith('[Sampling]')) {
|
|
181
|
+
return false;
|
|
182
|
+
}
|
|
183
|
+
return message.includes('no finite candidate logits after masking the pad token')
|
|
184
|
+
|| message.includes('Softmax produced no finite candidate probabilities');
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export function createUnhandledFinitenessPolicyError(state, contextLabel, error) {
|
|
188
|
+
const activePlan = resolveActiveExecutionPlan(state);
|
|
189
|
+
const wrapped = new Error(
|
|
190
|
+
`[Pipeline] ${contextLabel}: finiteness guard triggered for kernelPath ` +
|
|
191
|
+
`"${activePlan.kernelPathId ?? 'none'}" under fail-fast policy. ` +
|
|
192
|
+
'Resolve the unstable path with an explicit capability-aware execution override, ' +
|
|
193
|
+
'or opt into alternate-plan recovery with ' +
|
|
194
|
+
'runtime.inference.compute.rangeAwareSelectiveWidening.onTrigger="fallback-plan".',
|
|
195
|
+
error instanceof Error ? { cause: error } : undefined
|
|
196
|
+
);
|
|
197
|
+
wrapped.name = error?.name === 'FinitenessError' ? error.name : 'FinitenessError';
|
|
198
|
+
return wrapped;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function assertResolvedKVDtype(kvDtype, contextLabel) {
|
|
202
|
+
if (kvDtype === 'f16' || kvDtype === 'f32') {
|
|
203
|
+
return kvDtype;
|
|
204
|
+
}
|
|
205
|
+
throw new Error(
|
|
206
|
+
`[Pipeline] ${contextLabel}: expected execution-plan kvDtype to resolve to "f16" or "f32", ` +
|
|
207
|
+
`got ${JSON.stringify(kvDtype)}.`
|
|
208
|
+
);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export function resolveTargetPlanKVDtype(plan, contextLabel) {
|
|
212
|
+
return assertResolvedKVDtype(
|
|
213
|
+
plan?.kernelPath?.kvDtype ?? plan?.activationDtype ?? null,
|
|
214
|
+
contextLabel
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
export function resolveCurrentKVCacheDtype(state, plan, contextLabel) {
|
|
219
|
+
return assertResolvedKVDtype(
|
|
220
|
+
state?.kvCache?.kvDtype
|
|
221
|
+
?? plan?.kernelPath?.kvDtype
|
|
222
|
+
?? state?.runtimeConfig?.inference?.session?.kvcache?.kvDtype
|
|
223
|
+
?? plan?.activationDtype
|
|
224
|
+
?? null,
|
|
225
|
+
contextLabel
|
|
226
|
+
);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export function cloneRuntimeInferenceWithKVDtype(state, kvDtype) {
|
|
230
|
+
const runtimeInference = state?.runtimeConfig?.inference;
|
|
231
|
+
if (!runtimeInference?.session?.kvcache) {
|
|
232
|
+
throw new Error(
|
|
233
|
+
'[Pipeline] runtime.inference.session.kvcache is required for finiteness fallback KV-cache recovery.'
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
return mergeRuntimeValues(runtimeInference, {
|
|
237
|
+
session: {
|
|
238
|
+
kvcache: {
|
|
239
|
+
kvDtype,
|
|
240
|
+
},
|
|
241
|
+
},
|
|
242
|
+
});
|
|
243
|
+
}
|
|
@@ -9,6 +9,7 @@ import type { LayerContext } from './types.js';
|
|
|
9
9
|
import type { LogitsConfig, LogitsWeights } from './logits/index.js';
|
|
10
10
|
import type { WeightBufferConfig } from './weights.js';
|
|
11
11
|
import type { ExecutionSessionPlan } from './execution-plan.js';
|
|
12
|
+
import type { ExecutionV1PerLayerInputsSessionSchema } from '../../../config/schema/execution-v1.schema.js';
|
|
12
13
|
|
|
13
14
|
export declare function debugCheckBuffer(
|
|
14
15
|
state: PipelineState,
|
|
@@ -27,6 +28,11 @@ export declare function buildLayerContext(
|
|
|
27
28
|
executionPlan?: ExecutionSessionPlan | null
|
|
28
29
|
): LayerContext;
|
|
29
30
|
|
|
31
|
+
export declare function resolvePerLayerInputsSession(
|
|
32
|
+
manifestSession: ExecutionV1PerLayerInputsSessionSchema | null | undefined,
|
|
33
|
+
runtimeSession: ExecutionV1PerLayerInputsSessionSchema | Record<string, unknown> | null | undefined
|
|
34
|
+
): ExecutionV1PerLayerInputsSessionSchema | null;
|
|
35
|
+
|
|
30
36
|
export declare function releaseSharedAttentionState(
|
|
31
37
|
sharedAttentionState: LayerContext['sharedAttentionState'],
|
|
32
38
|
recorder?: {
|
|
@@ -34,8 +40,6 @@ export declare function releaseSharedAttentionState(
|
|
|
34
40
|
} | null
|
|
35
41
|
): void;
|
|
36
42
|
|
|
37
|
-
export declare function getWeightBufferConfig(state: PipelineState): WeightBufferConfig;
|
|
38
|
-
|
|
39
43
|
export declare function getLogitsWeights(state: PipelineState): LogitsWeights;
|
|
40
44
|
|
|
41
45
|
export declare function getLogitsConfig(state: PipelineState): LogitsConfig;
|
|
@@ -2,11 +2,23 @@
|
|
|
2
2
|
import { readBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
3
3
|
import { allowReadback } from '../../../gpu/perf-guards.js';
|
|
4
4
|
import { log } from '../../../debug/index.js';
|
|
5
|
+
import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
|
|
5
6
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
6
7
|
import { decodeReadback } from './debug-utils/index.js';
|
|
7
8
|
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
|
|
8
9
|
import { resolveRangeAwareSelectiveWideningConfig } from './finiteness-policy.js';
|
|
9
10
|
import { resolveActiveExecutionPlan } from './execution-plan.js';
|
|
11
|
+
import { isPlainObject } from '../../../utils/plain-object.js';
|
|
12
|
+
|
|
13
|
+
export function resolvePerLayerInputsSession(manifestSession, runtimeSession) {
|
|
14
|
+
if (!isPlainObject(runtimeSession)) {
|
|
15
|
+
return runtimeSession ?? (manifestSession ?? null);
|
|
16
|
+
}
|
|
17
|
+
if (!isPlainObject(manifestSession)) {
|
|
18
|
+
return runtimeSession;
|
|
19
|
+
}
|
|
20
|
+
return mergeRuntimeValues(manifestSession, runtimeSession);
|
|
21
|
+
}
|
|
10
22
|
export async function debugCheckBuffer(state, buffer, label, numTokens, expectedDim) {
|
|
11
23
|
if (!allowReadback(`pipeline.debug.${label}`)) return;
|
|
12
24
|
|
|
@@ -83,6 +95,10 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
|
|
|
83
95
|
const activeExecutionPlan = executionPlan ?? resolveActiveExecutionPlan(state);
|
|
84
96
|
const activeKernelPath = activeExecutionPlan.kernelPath ?? state.resolvedKernelPath ?? null;
|
|
85
97
|
const effectiveActivationDtype = activeExecutionPlan.activationDtype;
|
|
98
|
+
const perLayerInputsSession = resolvePerLayerInputsSession(
|
|
99
|
+
config.perLayerInputsSession ?? null,
|
|
100
|
+
state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
101
|
+
);
|
|
86
102
|
const effectiveComputeConfig = {
|
|
87
103
|
...computeConfig,
|
|
88
104
|
activationDtype: effectiveActivationDtype,
|
|
@@ -122,7 +138,7 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
|
|
|
122
138
|
debugProbes: state.runtimeConfig.shared.debug.probes,
|
|
123
139
|
debugCheckBuffer: debugCheckBufferFn,
|
|
124
140
|
perLayerInputBuffer: null,
|
|
125
|
-
perLayerInputsSession
|
|
141
|
+
perLayerInputsSession,
|
|
126
142
|
pipelinePlan: state.layerPipelinePlan,
|
|
127
143
|
expertWeights: state.expertWeights,
|
|
128
144
|
expertLoader: state.dopplerLoader,
|
|
@@ -142,6 +158,7 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
|
|
|
142
158
|
step: state.decodeStepCount,
|
|
143
159
|
phase: isDecodeMode ? 'decode' : 'prefill',
|
|
144
160
|
operatorDiagnostics: state.operatorDiagnostics,
|
|
161
|
+
useFusedGateUpGelu: state.runtimeConfig?.inference?.session?.useFusedGateUpGelu === true,
|
|
145
162
|
};
|
|
146
163
|
}
|
|
147
164
|
|
|
@@ -172,7 +189,7 @@ export function releaseSharedAttentionState(sharedAttentionState, recorder = nul
|
|
|
172
189
|
}
|
|
173
190
|
|
|
174
191
|
|
|
175
|
-
|
|
192
|
+
function getWeightBufferConfig(state) {
|
|
176
193
|
return {
|
|
177
194
|
rmsNormWeightOffset: state.modelConfig.rmsNormWeightOffset,
|
|
178
195
|
};
|
|
@@ -209,5 +226,6 @@ export function getLogitsConfig(state) {
|
|
|
209
226
|
activationDtype: effectiveActivationDtype,
|
|
210
227
|
kernelPath: activeKernelPath,
|
|
211
228
|
executionPolicies: state.executionV1State?.policies ?? null,
|
|
229
|
+
debugProbes: state.runtimeConfig.shared.debug.probes,
|
|
212
230
|
};
|
|
213
231
|
}
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
|
|
2
|
+
import { getDevice } from '../../../gpu/device.js';
|
|
3
|
+
import { acquireBuffer, releaseBuffer, uploadData } from '../../../memory/buffer-pool.js';
|
|
4
|
+
import { castF32ToF16 } from '../../../gpu/kernels/cast.js';
|
|
5
|
+
import { f32ToF16Array } from '../../kv-cache/types.js';
|
|
6
|
+
import { isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
|
|
7
|
+
import { createTensor } from '../../../gpu/tensor.js';
|
|
8
|
+
import { assertImplicitDtypeTransitionAllowed } from './dtype-contract.js';
|
|
9
|
+
import { applyChatTemplate } from './init.js';
|
|
10
|
+
import { formatChatMessages } from './chat-format.js';
|
|
11
|
+
|
|
12
|
+
function isStructuredChatRequest(prompt) {
|
|
13
|
+
return prompt != null
|
|
14
|
+
&& typeof prompt === 'object'
|
|
15
|
+
&& !Array.isArray(prompt)
|
|
16
|
+
&& Array.isArray(prompt.messages);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
|
|
20
|
+
const chatOptions = state.modelConfig.chatTemplateThinking === true ? { thinking: true } : undefined;
|
|
21
|
+
if (typeof prompt === 'string') {
|
|
22
|
+
if (useChatTemplate && state.modelConfig.chatTemplateType) {
|
|
23
|
+
if (state.modelConfig.chatTemplateType === 'translategemma') {
|
|
24
|
+
throw new Error(
|
|
25
|
+
`[Pipeline] ${contextLabel}: translategemma chat template requires structured messages. ` +
|
|
26
|
+
'Pass { messages: [...] } instead of a plain string prompt.'
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
return applyChatTemplate(prompt, state.modelConfig.chatTemplateType, chatOptions);
|
|
30
|
+
}
|
|
31
|
+
return prompt;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
if (prompt != null && typeof prompt === 'object' && !Array.isArray(prompt) && 'messages' in prompt && !Array.isArray(prompt.messages)) {
|
|
35
|
+
throw new Error(
|
|
36
|
+
`[Pipeline] ${contextLabel}: prompt.messages must be an array of chat messages, got ${typeof prompt.messages}. ` +
|
|
37
|
+
'Pass { messages: [{ role: "user", content: "..." }, ...] }.'
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
const messages = isStructuredChatRequest(prompt)
|
|
41
|
+
? prompt.messages
|
|
42
|
+
: (Array.isArray(prompt) ? prompt : null);
|
|
43
|
+
if (!messages) {
|
|
44
|
+
throw new Error(
|
|
45
|
+
`[Pipeline] ${contextLabel}: prompt must be a string, chat message array, or { messages: [...] }.`
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
const templateType = useChatTemplate ? state.modelConfig.chatTemplateType : null;
|
|
49
|
+
return formatChatMessages(messages, templateType, chatOptions);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function releasePerLayerInputBuffer(buffer, recorder, decodeBuffers, pleCache = null) {
|
|
53
|
+
if (!buffer) {
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
const ownsBuffer = decodeBuffers?.ownsBuffer(buffer) ?? false;
|
|
57
|
+
if (ownsBuffer) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const cachedPleBuffer = pleCache?.ownedBuffers instanceof Set && pleCache.ownedBuffers.has(buffer);
|
|
61
|
+
if (cachedPleBuffer) {
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
if (recorder) {
|
|
65
|
+
recorder.trackTemporaryBuffer(buffer);
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
releaseBuffer(buffer);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function normalizePrefixEmbeddingOverride(override, hiddenSize, numTokens, contextLabel) {
|
|
72
|
+
if (override == null) {
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
if (typeof override !== 'object') {
|
|
76
|
+
throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides must be an object when provided.`);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const prefixLength = Number(override.prefixLength);
|
|
80
|
+
if (!Number.isFinite(prefixLength) || prefixLength < 0 || Math.floor(prefixLength) !== prefixLength) {
|
|
81
|
+
throw new Error(
|
|
82
|
+
`[Pipeline] ${contextLabel}: embeddingOverrides.prefixLength must be a non-negative integer.`
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
if (prefixLength === 0) {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const offset = Number(override.offset ?? 0);
|
|
90
|
+
if (!Number.isFinite(offset) || offset < 0 || Math.floor(offset) !== offset) {
|
|
91
|
+
throw new Error(
|
|
92
|
+
`[Pipeline] ${contextLabel}: embeddingOverrides.offset must be a non-negative integer.`
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
if (offset + prefixLength > numTokens) {
|
|
96
|
+
throw new Error(
|
|
97
|
+
`[Pipeline] ${contextLabel}: embedding override offset=${offset} + prefixLength=${prefixLength} exceeds numTokens=${numTokens}.`
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const embeddings = override.embeddings ?? null;
|
|
102
|
+
if (!embeddings) {
|
|
103
|
+
throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides.embeddings is required when prefixLength > 0.`);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const expectedLength = prefixLength * hiddenSize;
|
|
107
|
+
if (embeddings instanceof Float32Array) {
|
|
108
|
+
if (embeddings.length !== expectedLength) {
|
|
109
|
+
throw new Error(
|
|
110
|
+
`[Pipeline] ${contextLabel}: embedding override length mismatch ` +
|
|
111
|
+
`(expected=${expectedLength}, got=${embeddings.length}).`
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
} else if (isGpuBufferInstance(embeddings)) {
|
|
115
|
+
const expectedBytes = expectedLength * Float32Array.BYTES_PER_ELEMENT;
|
|
116
|
+
if (embeddings.size < expectedBytes) {
|
|
117
|
+
throw new Error(
|
|
118
|
+
`[Pipeline] ${contextLabel}: embedding override GPUBuffer too small ` +
|
|
119
|
+
`(expected>=${expectedBytes} bytes, got=${embeddings.size}).`
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
} else {
|
|
123
|
+
throw new Error(
|
|
124
|
+
`[Pipeline] ${contextLabel}: embeddingOverrides.embeddings must be a Float32Array or GPUBuffer.`
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
return {
|
|
129
|
+
prefixLength,
|
|
130
|
+
offset,
|
|
131
|
+
embeddings,
|
|
132
|
+
expectedLength,
|
|
133
|
+
byteLength: expectedLength * Float32Array.BYTES_PER_ELEMENT,
|
|
134
|
+
byteOffset: offset * hiddenSize * Float32Array.BYTES_PER_ELEMENT,
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export function resolvePrefillEmbeddingInputIds(inputIds, embeddingInputSpan, contextLabel) {
|
|
139
|
+
if (embeddingInputSpan == null) {
|
|
140
|
+
return inputIds;
|
|
141
|
+
}
|
|
142
|
+
if (typeof embeddingInputSpan !== 'object') {
|
|
143
|
+
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan must be an object when provided.`);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const offset = Number(embeddingInputSpan.offset);
|
|
147
|
+
const length = Number(embeddingInputSpan.length);
|
|
148
|
+
const tokenId = Number(embeddingInputSpan.tokenId);
|
|
149
|
+
if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
|
|
150
|
+
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.offset must be a non-negative integer.`);
|
|
151
|
+
}
|
|
152
|
+
if (!Number.isFinite(length) || Math.floor(length) !== length || length < 0) {
|
|
153
|
+
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.length must be a non-negative integer.`);
|
|
154
|
+
}
|
|
155
|
+
if (!Number.isFinite(tokenId) || Math.floor(tokenId) !== tokenId || tokenId < 0) {
|
|
156
|
+
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.tokenId must be a non-negative integer.`);
|
|
157
|
+
}
|
|
158
|
+
if (offset + length > inputIds.length) {
|
|
159
|
+
throw new Error(
|
|
160
|
+
`[Pipeline] ${contextLabel}: embeddingInputSpan offset=${offset} + length=${length} ` +
|
|
161
|
+
`exceeds inputIds length ${inputIds.length}.`
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
const replacedInputIds = Array.from(inputIds);
|
|
165
|
+
replacedInputIds.fill(tokenId, offset, offset + length);
|
|
166
|
+
return replacedInputIds;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export function resolvePrefillMultimodalBidirectionalSpan(inputIds, bidirectionalSpan, contextLabel) {
|
|
170
|
+
if (bidirectionalSpan == null) {
|
|
171
|
+
return null;
|
|
172
|
+
}
|
|
173
|
+
if (typeof bidirectionalSpan !== 'object') {
|
|
174
|
+
throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan must be an object when provided.`);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const offset = Number(bidirectionalSpan.offset);
|
|
178
|
+
const length = Number(bidirectionalSpan.length);
|
|
179
|
+
if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
|
|
180
|
+
throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.offset must be a non-negative integer.`);
|
|
181
|
+
}
|
|
182
|
+
if (!Number.isFinite(length) || Math.floor(length) !== length || length < 1) {
|
|
183
|
+
throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.length must be a positive integer.`);
|
|
184
|
+
}
|
|
185
|
+
if ((offset + length) > inputIds.length) {
|
|
186
|
+
throw new Error(
|
|
187
|
+
`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan offset=${offset} + length=${length} ` +
|
|
188
|
+
`exceeds inputIds length ${inputIds.length}.`
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
return { offset, length };
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
export async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenSize, contextLabel, executionPolicies = null) {
|
|
195
|
+
if (!override) {
|
|
196
|
+
return baseTensor;
|
|
197
|
+
}
|
|
198
|
+
if (baseTensor.dtype !== 'f32' && baseTensor.dtype !== 'f16') {
|
|
199
|
+
throw new Error(
|
|
200
|
+
`[Pipeline] ${contextLabel}: embedding overrides require f32 or f16 activations, got ${baseTensor.dtype}.`
|
|
201
|
+
);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
const device = getDevice();
|
|
205
|
+
if (!device) {
|
|
206
|
+
throw new Error(`[Pipeline] ${contextLabel}: GPU device is required for embedding overrides.`);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
const outputBuffer = acquireBuffer(baseTensor.buffer.size, undefined, 'prefill_embedding_override');
|
|
210
|
+
const targetElementOffset = override.offset * hiddenSize;
|
|
211
|
+
const targetByteOffset = targetElementOffset * (baseTensor.dtype === 'f16' ? 2 : 4);
|
|
212
|
+
try {
|
|
213
|
+
const encoder = device.createCommandEncoder();
|
|
214
|
+
encoder.copyBufferToBuffer(baseTensor.buffer, 0, outputBuffer, 0, baseTensor.buffer.size);
|
|
215
|
+
if (baseTensor.dtype === 'f32') {
|
|
216
|
+
if (isGpuBufferInstance(override.embeddings)) {
|
|
217
|
+
encoder.copyBufferToBuffer(override.embeddings, 0, outputBuffer, override.byteOffset, override.byteLength);
|
|
218
|
+
device.queue.submit([encoder.finish()]);
|
|
219
|
+
} else {
|
|
220
|
+
device.queue.submit([encoder.finish()]);
|
|
221
|
+
uploadData(outputBuffer, override.embeddings, override.byteOffset);
|
|
222
|
+
}
|
|
223
|
+
return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
if (isGpuBufferInstance(override.embeddings)) {
|
|
227
|
+
const overrideTensor = createTensor(
|
|
228
|
+
override.embeddings,
|
|
229
|
+
'f32',
|
|
230
|
+
[override.prefixLength, hiddenSize],
|
|
231
|
+
'prefill_embedding_override_f32'
|
|
232
|
+
);
|
|
233
|
+
assertImplicitDtypeTransitionAllowed({
|
|
234
|
+
executionPolicies,
|
|
235
|
+
fromDtype: 'f32',
|
|
236
|
+
toDtype: 'f16',
|
|
237
|
+
op: 'prefill_embedding_override',
|
|
238
|
+
detail: 'Prefix embedding override would pack GPU-provided f32 features into an f16 activation buffer.',
|
|
239
|
+
});
|
|
240
|
+
const castedOverride = await castF32ToF16(overrideTensor);
|
|
241
|
+
try {
|
|
242
|
+
encoder.copyBufferToBuffer(
|
|
243
|
+
castedOverride.buffer,
|
|
244
|
+
0,
|
|
245
|
+
outputBuffer,
|
|
246
|
+
targetByteOffset,
|
|
247
|
+
castedOverride.buffer.size
|
|
248
|
+
);
|
|
249
|
+
device.queue.submit([encoder.finish()]);
|
|
250
|
+
} finally {
|
|
251
|
+
releaseBuffer(castedOverride.buffer);
|
|
252
|
+
}
|
|
253
|
+
} else {
|
|
254
|
+
assertImplicitDtypeTransitionAllowed({
|
|
255
|
+
executionPolicies,
|
|
256
|
+
fromDtype: 'f32',
|
|
257
|
+
toDtype: 'f16',
|
|
258
|
+
op: 'prefill_embedding_override',
|
|
259
|
+
detail: 'Prefix embedding override would pack CPU-provided f32 features into an f16 activation buffer.',
|
|
260
|
+
});
|
|
261
|
+
const packedOverride = f32ToF16Array(override.embeddings);
|
|
262
|
+
device.queue.submit([encoder.finish()]);
|
|
263
|
+
uploadData(outputBuffer, packedOverride, targetByteOffset);
|
|
264
|
+
}
|
|
265
|
+
return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
|
|
266
|
+
} catch (error) {
|
|
267
|
+
releaseBuffer(outputBuffer);
|
|
268
|
+
throw error;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
export function shouldDisablePrefillCommandBatching(state, opts, multimodalBidirectionalSpan) {
|
|
273
|
+
if (
|
|
274
|
+
opts?.disableCommandBatching === true
|
|
275
|
+
|| opts?.debug === true
|
|
276
|
+
|| (Array.isArray(opts?.debugLayers) && opts.debugLayers.length > 0)
|
|
277
|
+
) {
|
|
278
|
+
return true;
|
|
279
|
+
}
|
|
280
|
+
if (state?.kvCache?.layout === 'bdpa_paged') {
|
|
281
|
+
return true;
|
|
282
|
+
}
|
|
283
|
+
if (multimodalBidirectionalSpan == null) {
|
|
284
|
+
return false;
|
|
285
|
+
}
|
|
286
|
+
if (state?.kvCache?.hasGPUCache?.() !== true) {
|
|
287
|
+
return false;
|
|
288
|
+
}
|
|
289
|
+
// WORKAROUND: recorded prefill with live KV-cache writes regresses Gemma 4
|
|
290
|
+
// multimodal logits on the first generated token. Keep this lane fail-closed
|
|
291
|
+
// until the recorder/cache interaction is numerically matched to the direct path.
|
|
292
|
+
return true;
|
|
293
|
+
}
|
|
@@ -75,13 +75,6 @@ export function resolveFloatDtypeFromByteSize(
|
|
|
75
75
|
fallback?: 'f16' | 'f32'
|
|
76
76
|
): 'f16' | 'f32';
|
|
77
77
|
|
|
78
|
-
export function decodeFloatWeights(
|
|
79
|
-
data: ArrayBuffer,
|
|
80
|
-
dtype: 'f16' | 'f32' | 'bf16',
|
|
81
|
-
expectedLength: number,
|
|
82
|
-
label: string
|
|
83
|
-
): Float32Array;
|
|
84
|
-
|
|
85
78
|
export function getFinalNormWeights(state: PipelineState): Promise<Float32Array>;
|
|
86
79
|
|
|
87
80
|
export function extractEmbeddingFromHidden(
|
|
@@ -297,11 +297,13 @@ export function resolvePrefillEmbeddingOptions(state, options = {}) {
|
|
|
297
297
|
? state.manifest.modelType.toLowerCase()
|
|
298
298
|
: '';
|
|
299
299
|
const generationDefaults = state.runtimeConfig.inference.generation;
|
|
300
|
-
//
|
|
301
|
-
//
|
|
302
|
-
//
|
|
303
|
-
//
|
|
304
|
-
const
|
|
300
|
+
// Models that expose embedding extraction default to 'mean' pooling. This covers
|
|
301
|
+
// dedicated embedding models (modelType="embedding") and text-generation models
|
|
302
|
+
// that opt in via inference.supportsEmbedding=true. Conversion configs can still
|
|
303
|
+
// override via generation.embeddingMode in their runtime profile.
|
|
304
|
+
const supportsEmbeddingExtraction = modelType === 'embedding'
|
|
305
|
+
|| state.manifest?.inference?.supportsEmbedding === true;
|
|
306
|
+
const defaultEmbeddingMode = supportsEmbeddingExtraction
|
|
305
307
|
? 'mean'
|
|
306
308
|
: generationDefaults.embeddingMode;
|
|
307
309
|
return {
|
|
@@ -349,7 +351,7 @@ export function resolveFloatDtypeFromByteSize(totalBytes, expectedLength) {
|
|
|
349
351
|
});
|
|
350
352
|
}
|
|
351
353
|
|
|
352
|
-
|
|
354
|
+
function decodeFloatWeights(data, dtype, expectedLength, label) {
|
|
353
355
|
const decodeDtype = resolveFloatDtypeFromAlias(dtype);
|
|
354
356
|
const decoded = decodeReadback(data, decodeDtype);
|
|
355
357
|
if (decoded.length !== expectedLength) {
|