doppler-gpu 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +19 -15
- package/package.json +77 -29
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +64 -6
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +524 -350
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +12 -2
- package/src/config/schema/doppler.schema.js +108 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +68 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +76 -0
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +107 -0
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +3 -21
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +6 -5
- package/src/gpu/kernels/index.js +3 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +12 -0
- package/src/inference/browser-harness-text-helpers.js +223 -14
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +264 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +22 -27
- package/src/inference/pipelines/text/embed.js +0 -39
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +4 -5
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +8 -3
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +223 -547
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +5 -2
- package/src/inference/pipelines/text/init.js +183 -724
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.js +2 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +120 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +223 -27
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +13 -3
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +11 -3
- package/src/storage/artifact-storage-context.js +231 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +21 -24
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +2 -0
- package/src/tooling/source-runtime-bundle.js +3 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -14,15 +14,11 @@ import {
|
|
|
14
14
|
validateCaptureConfig,
|
|
15
15
|
} from '../../../debug/index.js';
|
|
16
16
|
import { validateCallTimeOptions } from '../../../config/param-validator.js';
|
|
17
|
-
import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
|
|
18
17
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
19
|
-
import { castF32ToF16 } from '../../../gpu/kernels/cast.js';
|
|
20
|
-
import { f32ToF16Array } from '../../kv-cache/types.js';
|
|
21
18
|
|
|
22
19
|
// Pipeline sub-modules
|
|
23
20
|
import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './sampling.js';
|
|
24
|
-
import {
|
|
25
|
-
import { formatChatMessages } from './chat-format.js';
|
|
21
|
+
import { createKVCache, isStopToken } from './init.js';
|
|
26
22
|
import { embed } from './embed.js';
|
|
27
23
|
import { processLayer } from './layer.js';
|
|
28
24
|
import { computeLogits, recordLogitsGPU, extractLastPositionLogits, applySoftcapping } from './logits/index.js';
|
|
@@ -62,12 +58,14 @@ import { resolveSamplingConfig } from './sampling-config.js';
|
|
|
62
58
|
import { decodeReadback, getLogitsHealth } from './debug-utils/index.js';
|
|
63
59
|
import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
|
|
64
60
|
import { resolveDeferredRoundingWindowTokens } from './finiteness-policy.js';
|
|
61
|
+
import { drainPendingTsirReads } from './tsir-fixture-writer.js';
|
|
65
62
|
import {
|
|
66
63
|
activateFallbackExecutionPlan,
|
|
67
64
|
hasFallbackExecutionPlan,
|
|
68
65
|
rebaseExecutionSessionPlan,
|
|
69
66
|
resetActiveExecutionPlan,
|
|
70
67
|
resolveMaxBatchDecodeTokens,
|
|
68
|
+
resolvePrefillRecorderChunkLayers,
|
|
71
69
|
resolveActiveExecutionPlan,
|
|
72
70
|
setActiveExecutionPlan,
|
|
73
71
|
} from './execution-plan.js';
|
|
@@ -81,15 +79,37 @@ import {
|
|
|
81
79
|
preparePerLayerInputs,
|
|
82
80
|
createPleBufferCache,
|
|
83
81
|
prefetchPerLayerRow,
|
|
84
|
-
ensurePleScaledProjectionNormWeight,
|
|
85
|
-
ensurePleGpuHotVocabularyRuntime,
|
|
86
|
-
ensurePleGpuSplitTablesRuntime,
|
|
87
82
|
getPleHotVocabularyRuntime,
|
|
88
83
|
hasGpuSplitPerLayerInputEmbeddings,
|
|
89
84
|
hasRangeBackedPerLayerInputEmbeddings,
|
|
90
85
|
} from './per-layer-inputs.js';
|
|
91
86
|
import { createTensor } from '../../../gpu/tensor.js';
|
|
92
|
-
|
|
87
|
+
|
|
88
|
+
// Extracted standalone helpers
|
|
89
|
+
import {
|
|
90
|
+
resolvePromptInput,
|
|
91
|
+
releasePerLayerInputBuffer,
|
|
92
|
+
normalizePrefixEmbeddingOverride,
|
|
93
|
+
resolvePrefillEmbeddingInputIds,
|
|
94
|
+
resolvePrefillMultimodalBidirectionalSpan,
|
|
95
|
+
applyPrefixEmbeddingOverride,
|
|
96
|
+
shouldDisablePrefillCommandBatching,
|
|
97
|
+
} from './generator-prefill-helpers.js';
|
|
98
|
+
import {
|
|
99
|
+
shouldDisableBatchDecodeAfterShortBatch,
|
|
100
|
+
resolveHotVocabularyBatchDecodeAvailability,
|
|
101
|
+
primePleDecodeRuntimeCache,
|
|
102
|
+
recordPrefillProfileStep,
|
|
103
|
+
resolveTokenText,
|
|
104
|
+
usesReplayPrefillDecode,
|
|
105
|
+
assertIncrementalDecodeSupport,
|
|
106
|
+
summarizeExecutionPlan,
|
|
107
|
+
shouldRetryWithFinitenessFallback,
|
|
108
|
+
createUnhandledFinitenessPolicyError,
|
|
109
|
+
resolveTargetPlanKVDtype,
|
|
110
|
+
resolveCurrentKVCacheDtype,
|
|
111
|
+
cloneRuntimeInferenceWithKVDtype,
|
|
112
|
+
} from './generator-decode-policy.js';
|
|
93
113
|
|
|
94
114
|
let intentBundleModulePromise = null;
|
|
95
115
|
|
|
@@ -98,521 +118,6 @@ async function getExperimentalIntentBundleModule() {
|
|
|
98
118
|
return intentBundleModulePromise;
|
|
99
119
|
}
|
|
100
120
|
|
|
101
|
-
function isStructuredChatRequest(prompt) {
|
|
102
|
-
return prompt != null
|
|
103
|
-
&& typeof prompt === 'object'
|
|
104
|
-
&& !Array.isArray(prompt)
|
|
105
|
-
&& Array.isArray(prompt.messages);
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
export function shouldDisableBatchDecodeAfterShortBatch({ hitStop, actualCount, requestedCount }) {
|
|
109
|
-
return hitStop !== true
|
|
110
|
-
&& Number.isInteger(actualCount)
|
|
111
|
-
&& Number.isInteger(requestedCount)
|
|
112
|
-
&& actualCount > 0
|
|
113
|
-
&& actualCount < requestedCount;
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
export function resolveHotVocabularyBatchDecodeAvailability({
|
|
117
|
-
hasRangeBackedPerLayerInputs,
|
|
118
|
-
pleHotVocabularyRuntime,
|
|
119
|
-
tokenId,
|
|
120
|
-
}) {
|
|
121
|
-
if (hasRangeBackedPerLayerInputs !== true) {
|
|
122
|
-
return false;
|
|
123
|
-
}
|
|
124
|
-
if (!pleHotVocabularyRuntime || typeof pleHotVocabularyRuntime !== 'object') {
|
|
125
|
-
return false;
|
|
126
|
-
}
|
|
127
|
-
const sentinelIndex = pleHotVocabularyRuntime.sentinelIndex;
|
|
128
|
-
if (!Number.isInteger(sentinelIndex)) {
|
|
129
|
-
return false;
|
|
130
|
-
}
|
|
131
|
-
if (!Number.isInteger(tokenId)) {
|
|
132
|
-
return false;
|
|
133
|
-
}
|
|
134
|
-
const hotTokenIndex = pleHotVocabularyRuntime.hotTokenIndexMap?.[tokenId] ?? sentinelIndex;
|
|
135
|
-
return hotTokenIndex !== sentinelIndex;
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
|
|
139
|
-
const perLayerInputsSession = resolvePerLayerInputsSession(
|
|
140
|
-
state.modelConfig?.perLayerInputsSession ?? null,
|
|
141
|
-
state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
142
|
-
);
|
|
143
|
-
const materialization = perLayerInputsSession?.materialization;
|
|
144
|
-
if (state.debug) {
|
|
145
|
-
log.debug(
|
|
146
|
-
'Pipeline',
|
|
147
|
-
`PLE materialization session=${materialization ?? 'null'} ` +
|
|
148
|
-
`modelId=${state.modelConfig?.modelId ?? 'unknown'} ` +
|
|
149
|
-
`runtimePerLayerInputs=${Boolean(state.runtimeConfig.inference.session?.perLayerInputs)} ` +
|
|
150
|
-
`manifestPerLayerInputs=${Boolean(state.modelConfig?.perLayerInputsSession)}`
|
|
151
|
-
);
|
|
152
|
-
}
|
|
153
|
-
await ensurePleGpuHotVocabularyRuntime({
|
|
154
|
-
config: state.modelConfig,
|
|
155
|
-
weights: state.weights,
|
|
156
|
-
perLayerInputsSession,
|
|
157
|
-
debugFlags: state.debugFlags,
|
|
158
|
-
tokenizer: state.tokenizer ?? null,
|
|
159
|
-
seedTokenIds: Array.isArray(seedTokenIds) ? seedTokenIds : null,
|
|
160
|
-
});
|
|
161
|
-
await ensurePleGpuSplitTablesRuntime({
|
|
162
|
-
config: state.modelConfig,
|
|
163
|
-
weights: state.weights,
|
|
164
|
-
perLayerInputsSession,
|
|
165
|
-
debugFlags: state.debugFlags,
|
|
166
|
-
});
|
|
167
|
-
await ensurePleScaledProjectionNormWeight({
|
|
168
|
-
config: state.modelConfig,
|
|
169
|
-
weights: state.weights,
|
|
170
|
-
weightConfig: {
|
|
171
|
-
rmsNormWeightOffset: state.modelConfig?.rmsNormWeightOffset ?? false,
|
|
172
|
-
},
|
|
173
|
-
debugFlags: state.debugFlags,
|
|
174
|
-
});
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
function recordPrefillProfileStep(state, entry) {
|
|
178
|
-
if (!entry?.timings || Object.keys(entry.timings).length === 0) return;
|
|
179
|
-
if (!state.stats.prefillProfileSteps) {
|
|
180
|
-
state.stats.prefillProfileSteps = [];
|
|
181
|
-
}
|
|
182
|
-
state.stats.prefillProfileSteps.push(entry);
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
|
|
186
|
-
const chatOptions = state.modelConfig.chatTemplateThinking === true ? { thinking: true } : undefined;
|
|
187
|
-
if (typeof prompt === 'string') {
|
|
188
|
-
if (useChatTemplate && state.modelConfig.chatTemplateType) {
|
|
189
|
-
if (state.modelConfig.chatTemplateType === 'translategemma') {
|
|
190
|
-
throw new Error(
|
|
191
|
-
`[Pipeline] ${contextLabel}: translategemma chat template requires structured messages. ` +
|
|
192
|
-
'Pass { messages: [...] } instead of a plain string prompt.'
|
|
193
|
-
);
|
|
194
|
-
}
|
|
195
|
-
return applyChatTemplate(prompt, state.modelConfig.chatTemplateType, chatOptions);
|
|
196
|
-
}
|
|
197
|
-
return prompt;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
if (prompt != null && typeof prompt === 'object' && !Array.isArray(prompt) && 'messages' in prompt && !Array.isArray(prompt.messages)) {
|
|
201
|
-
throw new Error(
|
|
202
|
-
`[Pipeline] ${contextLabel}: prompt.messages must be an array of chat messages, got ${typeof prompt.messages}. ` +
|
|
203
|
-
'Pass { messages: [{ role: "user", content: "..." }, ...] }.'
|
|
204
|
-
);
|
|
205
|
-
}
|
|
206
|
-
const messages = isStructuredChatRequest(prompt)
|
|
207
|
-
? prompt.messages
|
|
208
|
-
: (Array.isArray(prompt) ? prompt : null);
|
|
209
|
-
if (!messages) {
|
|
210
|
-
throw new Error(
|
|
211
|
-
`[Pipeline] ${contextLabel}: prompt must be a string, chat message array, or { messages: [...] }.`
|
|
212
|
-
);
|
|
213
|
-
}
|
|
214
|
-
const templateType = useChatTemplate ? state.modelConfig.chatTemplateType : null;
|
|
215
|
-
return formatChatMessages(messages, templateType, chatOptions);
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
function releasePerLayerInputBuffer(buffer, recorder, decodeBuffers, pleCache = null) {
|
|
219
|
-
if (!buffer) {
|
|
220
|
-
return;
|
|
221
|
-
}
|
|
222
|
-
const ownsBuffer = decodeBuffers?.ownsBuffer(buffer) ?? false;
|
|
223
|
-
if (ownsBuffer) {
|
|
224
|
-
return;
|
|
225
|
-
}
|
|
226
|
-
const cachedPleBuffer = pleCache?.ownedBuffers instanceof Set && pleCache.ownedBuffers.has(buffer);
|
|
227
|
-
if (cachedPleBuffer) {
|
|
228
|
-
return;
|
|
229
|
-
}
|
|
230
|
-
if (recorder) {
|
|
231
|
-
recorder.trackTemporaryBuffer(buffer);
|
|
232
|
-
return;
|
|
233
|
-
}
|
|
234
|
-
releaseBuffer(buffer);
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
function normalizePrefixEmbeddingOverride(override, hiddenSize, numTokens, contextLabel) {
|
|
238
|
-
if (override == null) {
|
|
239
|
-
return null;
|
|
240
|
-
}
|
|
241
|
-
if (typeof override !== 'object') {
|
|
242
|
-
throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides must be an object when provided.`);
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
const prefixLength = Number(override.prefixLength);
|
|
246
|
-
if (!Number.isFinite(prefixLength) || prefixLength < 0 || Math.floor(prefixLength) !== prefixLength) {
|
|
247
|
-
throw new Error(
|
|
248
|
-
`[Pipeline] ${contextLabel}: embeddingOverrides.prefixLength must be a non-negative integer.`
|
|
249
|
-
);
|
|
250
|
-
}
|
|
251
|
-
if (prefixLength === 0) {
|
|
252
|
-
return null;
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
const offset = Number(override.offset ?? 0);
|
|
256
|
-
if (!Number.isFinite(offset) || offset < 0 || Math.floor(offset) !== offset) {
|
|
257
|
-
throw new Error(
|
|
258
|
-
`[Pipeline] ${contextLabel}: embeddingOverrides.offset must be a non-negative integer.`
|
|
259
|
-
);
|
|
260
|
-
}
|
|
261
|
-
if (offset + prefixLength > numTokens) {
|
|
262
|
-
throw new Error(
|
|
263
|
-
`[Pipeline] ${contextLabel}: embedding override offset=${offset} + prefixLength=${prefixLength} exceeds numTokens=${numTokens}.`
|
|
264
|
-
);
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
const embeddings = override.embeddings ?? null;
|
|
268
|
-
if (!embeddings) {
|
|
269
|
-
throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides.embeddings is required when prefixLength > 0.`);
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
const expectedLength = prefixLength * hiddenSize;
|
|
273
|
-
if (embeddings instanceof Float32Array) {
|
|
274
|
-
if (embeddings.length !== expectedLength) {
|
|
275
|
-
throw new Error(
|
|
276
|
-
`[Pipeline] ${contextLabel}: embedding override length mismatch ` +
|
|
277
|
-
`(expected=${expectedLength}, got=${embeddings.length}).`
|
|
278
|
-
);
|
|
279
|
-
}
|
|
280
|
-
} else if (isGpuBufferInstance(embeddings)) {
|
|
281
|
-
const expectedBytes = expectedLength * Float32Array.BYTES_PER_ELEMENT;
|
|
282
|
-
if (embeddings.size < expectedBytes) {
|
|
283
|
-
throw new Error(
|
|
284
|
-
`[Pipeline] ${contextLabel}: embedding override GPUBuffer too small ` +
|
|
285
|
-
`(expected>=${expectedBytes} bytes, got=${embeddings.size}).`
|
|
286
|
-
);
|
|
287
|
-
}
|
|
288
|
-
} else {
|
|
289
|
-
throw new Error(
|
|
290
|
-
`[Pipeline] ${contextLabel}: embeddingOverrides.embeddings must be a Float32Array or GPUBuffer.`
|
|
291
|
-
);
|
|
292
|
-
}
|
|
293
|
-
|
|
294
|
-
return {
|
|
295
|
-
prefixLength,
|
|
296
|
-
offset,
|
|
297
|
-
embeddings,
|
|
298
|
-
expectedLength,
|
|
299
|
-
byteLength: expectedLength * Float32Array.BYTES_PER_ELEMENT,
|
|
300
|
-
byteOffset: offset * hiddenSize * Float32Array.BYTES_PER_ELEMENT,
|
|
301
|
-
};
|
|
302
|
-
}
|
|
303
|
-
|
|
304
|
-
function resolvePrefillEmbeddingInputIds(inputIds, embeddingInputSpan, contextLabel) {
|
|
305
|
-
if (embeddingInputSpan == null) {
|
|
306
|
-
return inputIds;
|
|
307
|
-
}
|
|
308
|
-
if (typeof embeddingInputSpan !== 'object') {
|
|
309
|
-
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan must be an object when provided.`);
|
|
310
|
-
}
|
|
311
|
-
|
|
312
|
-
const offset = Number(embeddingInputSpan.offset);
|
|
313
|
-
const length = Number(embeddingInputSpan.length);
|
|
314
|
-
const tokenId = Number(embeddingInputSpan.tokenId);
|
|
315
|
-
if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
|
|
316
|
-
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.offset must be a non-negative integer.`);
|
|
317
|
-
}
|
|
318
|
-
if (!Number.isFinite(length) || Math.floor(length) !== length || length < 0) {
|
|
319
|
-
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.length must be a non-negative integer.`);
|
|
320
|
-
}
|
|
321
|
-
if (!Number.isFinite(tokenId) || Math.floor(tokenId) !== tokenId || tokenId < 0) {
|
|
322
|
-
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.tokenId must be a non-negative integer.`);
|
|
323
|
-
}
|
|
324
|
-
if (offset + length > inputIds.length) {
|
|
325
|
-
throw new Error(
|
|
326
|
-
`[Pipeline] ${contextLabel}: embeddingInputSpan offset=${offset} + length=${length} ` +
|
|
327
|
-
`exceeds inputIds length ${inputIds.length}.`
|
|
328
|
-
);
|
|
329
|
-
}
|
|
330
|
-
const replacedInputIds = Array.from(inputIds);
|
|
331
|
-
replacedInputIds.fill(tokenId, offset, offset + length);
|
|
332
|
-
return replacedInputIds;
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
function resolvePrefillMultimodalBidirectionalSpan(inputIds, bidirectionalSpan, contextLabel) {
|
|
336
|
-
if (bidirectionalSpan == null) {
|
|
337
|
-
return null;
|
|
338
|
-
}
|
|
339
|
-
if (typeof bidirectionalSpan !== 'object') {
|
|
340
|
-
throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan must be an object when provided.`);
|
|
341
|
-
}
|
|
342
|
-
|
|
343
|
-
const offset = Number(bidirectionalSpan.offset);
|
|
344
|
-
const length = Number(bidirectionalSpan.length);
|
|
345
|
-
if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
|
|
346
|
-
throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.offset must be a non-negative integer.`);
|
|
347
|
-
}
|
|
348
|
-
if (!Number.isFinite(length) || Math.floor(length) !== length || length < 1) {
|
|
349
|
-
throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.length must be a positive integer.`);
|
|
350
|
-
}
|
|
351
|
-
if ((offset + length) > inputIds.length) {
|
|
352
|
-
throw new Error(
|
|
353
|
-
`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan offset=${offset} + length=${length} ` +
|
|
354
|
-
`exceeds inputIds length ${inputIds.length}.`
|
|
355
|
-
);
|
|
356
|
-
}
|
|
357
|
-
return { offset, length };
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenSize, contextLabel, executionPolicies = null) {
|
|
361
|
-
if (!override) {
|
|
362
|
-
return baseTensor;
|
|
363
|
-
}
|
|
364
|
-
if (baseTensor.dtype !== 'f32' && baseTensor.dtype !== 'f16') {
|
|
365
|
-
throw new Error(
|
|
366
|
-
`[Pipeline] ${contextLabel}: embedding overrides require f32 or f16 activations, got ${baseTensor.dtype}.`
|
|
367
|
-
);
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
const device = getDevice();
|
|
371
|
-
if (!device) {
|
|
372
|
-
throw new Error(`[Pipeline] ${contextLabel}: GPU device is required for embedding overrides.`);
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
const outputBuffer = acquireBuffer(baseTensor.buffer.size, undefined, 'prefill_embedding_override');
|
|
376
|
-
const targetElementOffset = override.offset * hiddenSize;
|
|
377
|
-
const targetByteOffset = targetElementOffset * (baseTensor.dtype === 'f16' ? 2 : 4);
|
|
378
|
-
try {
|
|
379
|
-
const encoder = device.createCommandEncoder();
|
|
380
|
-
encoder.copyBufferToBuffer(baseTensor.buffer, 0, outputBuffer, 0, baseTensor.buffer.size);
|
|
381
|
-
if (baseTensor.dtype === 'f32') {
|
|
382
|
-
if (isGpuBufferInstance(override.embeddings)) {
|
|
383
|
-
encoder.copyBufferToBuffer(override.embeddings, 0, outputBuffer, override.byteOffset, override.byteLength);
|
|
384
|
-
device.queue.submit([encoder.finish()]);
|
|
385
|
-
} else {
|
|
386
|
-
device.queue.submit([encoder.finish()]);
|
|
387
|
-
uploadData(outputBuffer, override.embeddings, override.byteOffset);
|
|
388
|
-
}
|
|
389
|
-
return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
if (isGpuBufferInstance(override.embeddings)) {
|
|
393
|
-
const overrideTensor = createTensor(
|
|
394
|
-
override.embeddings,
|
|
395
|
-
'f32',
|
|
396
|
-
[override.prefixLength, hiddenSize],
|
|
397
|
-
'prefill_embedding_override_f32'
|
|
398
|
-
);
|
|
399
|
-
assertImplicitDtypeTransitionAllowed({
|
|
400
|
-
executionPolicies,
|
|
401
|
-
fromDtype: 'f32',
|
|
402
|
-
toDtype: 'f16',
|
|
403
|
-
op: 'prefill_embedding_override',
|
|
404
|
-
detail: 'Prefix embedding override would pack GPU-provided f32 features into an f16 activation buffer.',
|
|
405
|
-
});
|
|
406
|
-
const castedOverride = await castF32ToF16(overrideTensor);
|
|
407
|
-
try {
|
|
408
|
-
encoder.copyBufferToBuffer(
|
|
409
|
-
castedOverride.buffer,
|
|
410
|
-
0,
|
|
411
|
-
outputBuffer,
|
|
412
|
-
targetByteOffset,
|
|
413
|
-
castedOverride.buffer.size
|
|
414
|
-
);
|
|
415
|
-
device.queue.submit([encoder.finish()]);
|
|
416
|
-
} finally {
|
|
417
|
-
releaseBuffer(castedOverride.buffer);
|
|
418
|
-
}
|
|
419
|
-
} else {
|
|
420
|
-
assertImplicitDtypeTransitionAllowed({
|
|
421
|
-
executionPolicies,
|
|
422
|
-
fromDtype: 'f32',
|
|
423
|
-
toDtype: 'f16',
|
|
424
|
-
op: 'prefill_embedding_override',
|
|
425
|
-
detail: 'Prefix embedding override would pack CPU-provided f32 features into an f16 activation buffer.',
|
|
426
|
-
});
|
|
427
|
-
const packedOverride = f32ToF16Array(override.embeddings);
|
|
428
|
-
device.queue.submit([encoder.finish()]);
|
|
429
|
-
uploadData(outputBuffer, packedOverride, targetByteOffset);
|
|
430
|
-
}
|
|
431
|
-
return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
|
|
432
|
-
} catch (error) {
|
|
433
|
-
releaseBuffer(outputBuffer);
|
|
434
|
-
throw error;
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
/**
|
|
439
|
-
* Resolve display text for a token ID sequence.
|
|
440
|
-
*
|
|
441
|
-
* Preference order:
|
|
442
|
-
* 1. renderTokenText (custom primary renderer) — if provided and returns non-empty string.
|
|
443
|
-
* 2. tokenizer.decode (default primary) — if renderTokenText is not provided.
|
|
444
|
-
* 3. renderFallbackTokenText (custom fallback) — if provided and returns non-empty string,
|
|
445
|
-
* unless primary returned empty string and fallback looks like a special token (<...>),
|
|
446
|
-
* in which case empty string is preserved to keep skip-special behavior deterministic.
|
|
447
|
-
* 4. tokenizer.decode with skipSpecialTokens=false (default fallback) — same special-token guard.
|
|
448
|
-
* 5. fallbackText — static fallback string (default '?').
|
|
449
|
-
*/
|
|
450
|
-
function resolveTokenText(tokenizer, tokenIds, fallbackText = '?', renderTokenText, renderFallbackTokenText) {
|
|
451
|
-
const renderPrimary = typeof renderTokenText === 'function'
|
|
452
|
-
? renderTokenText
|
|
453
|
-
: (ids) => tokenizer?.decode?.(ids);
|
|
454
|
-
const renderFallback = typeof renderFallbackTokenText === 'function'
|
|
455
|
-
? renderFallbackTokenText
|
|
456
|
-
: (ids) => tokenizer?.decode?.(ids, false);
|
|
457
|
-
|
|
458
|
-
const primaryText = renderPrimary(tokenIds);
|
|
459
|
-
if (typeof primaryText === 'string' && primaryText.length > 0) {
|
|
460
|
-
return primaryText;
|
|
461
|
-
}
|
|
462
|
-
|
|
463
|
-
const fallback = renderFallback(tokenIds);
|
|
464
|
-
if (typeof fallback === 'string' && fallback.length > 0) {
|
|
465
|
-
// Keep skip-special behavior deterministic: if primary decoding filtered this
|
|
466
|
-
// token to empty, do not reintroduce obvious special-token text via fallback.
|
|
467
|
-
if (
|
|
468
|
-
primaryText === ''
|
|
469
|
-
&& /^<[^>\n]{1,80}>$/.test(fallback.trim())
|
|
470
|
-
) {
|
|
471
|
-
return '';
|
|
472
|
-
}
|
|
473
|
-
return fallback;
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
return fallbackText;
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
function usesReplayPrefillDecode(state) {
|
|
480
|
-
return state?.modelConfig?.decodeStrategy === 'replay_prefill';
|
|
481
|
-
}
|
|
482
|
-
|
|
483
|
-
export function shouldDisablePrefillCommandBatching(state, opts, multimodalBidirectionalSpan) {
|
|
484
|
-
if (
|
|
485
|
-
opts?.disableCommandBatching === true
|
|
486
|
-
|| opts?.debug === true
|
|
487
|
-
|| (Array.isArray(opts?.debugLayers) && opts.debugLayers.length > 0)
|
|
488
|
-
) {
|
|
489
|
-
return true;
|
|
490
|
-
}
|
|
491
|
-
if (state?.kvCache?.layout === 'bdpa_paged') {
|
|
492
|
-
return true;
|
|
493
|
-
}
|
|
494
|
-
if (multimodalBidirectionalSpan == null) {
|
|
495
|
-
return false;
|
|
496
|
-
}
|
|
497
|
-
if (state?.kvCache?.hasGPUCache?.() !== true) {
|
|
498
|
-
return false;
|
|
499
|
-
}
|
|
500
|
-
// WORKAROUND: recorded prefill with live KV-cache writes regresses Gemma 4
|
|
501
|
-
// multimodal logits on the first generated token. Keep this lane fail-closed
|
|
502
|
-
// until the recorder/cache interaction is numerically matched to the direct path.
|
|
503
|
-
return true;
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
function assertIncrementalDecodeSupport(state, operation) {
|
|
507
|
-
if (!usesReplayPrefillDecode(state)) {
|
|
508
|
-
return;
|
|
509
|
-
}
|
|
510
|
-
throw new Error(
|
|
511
|
-
`[Pipeline] ${operation} is not supported for models that require replay-prefill decode. ` +
|
|
512
|
-
'Incremental KV-cache decode and prefix snapshots stay disabled when the model config does not resolve ' +
|
|
513
|
-
'explicit layerTypes for mixed-geometry/shared-KV decode.'
|
|
514
|
-
);
|
|
515
|
-
}
|
|
516
|
-
|
|
517
|
-
function summarizeExecutionPlan(plan) {
|
|
518
|
-
if (!plan) {
|
|
519
|
-
return null;
|
|
520
|
-
}
|
|
521
|
-
if (typeof plan !== 'object') {
|
|
522
|
-
log.warn('Pipeline', `summarizeExecutionPlan: expected object, got ${typeof plan}`);
|
|
523
|
-
return null;
|
|
524
|
-
}
|
|
525
|
-
if (typeof plan.id !== 'string') {
|
|
526
|
-
log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "id"');
|
|
527
|
-
}
|
|
528
|
-
if (typeof plan.activationDtype !== 'string') {
|
|
529
|
-
log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "activationDtype"');
|
|
530
|
-
}
|
|
531
|
-
return {
|
|
532
|
-
id: plan.id,
|
|
533
|
-
kernelPathId: plan.kernelPathId ?? null,
|
|
534
|
-
kernelPathSource: plan.kernelPathSource ?? 'none',
|
|
535
|
-
activationDtype: plan.activationDtype,
|
|
536
|
-
readbackInterval: plan.readbackInterval ?? null,
|
|
537
|
-
batchSize: plan.defaultBatchSize,
|
|
538
|
-
stopCheckMode: plan.defaultStopCheckMode,
|
|
539
|
-
ringTokens: plan.ringTokens ?? null,
|
|
540
|
-
ringStop: plan.ringStop ?? null,
|
|
541
|
-
ringStaging: plan.ringStaging ?? null,
|
|
542
|
-
};
|
|
543
|
-
}
|
|
544
|
-
|
|
545
|
-
export function shouldRetryWithFinitenessFallback(error) {
|
|
546
|
-
if (error?.name === 'FinitenessError') {
|
|
547
|
-
return true;
|
|
548
|
-
}
|
|
549
|
-
const message = typeof error?.message === 'string'
|
|
550
|
-
? error.message
|
|
551
|
-
: (typeof error === 'string' ? error : '');
|
|
552
|
-
if (!message.startsWith('[Sampling]')) {
|
|
553
|
-
return false;
|
|
554
|
-
}
|
|
555
|
-
return message.includes('no finite candidate logits after masking the pad token')
|
|
556
|
-
|| message.includes('Softmax produced no finite candidate probabilities');
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
function createUnhandledFinitenessPolicyError(state, contextLabel, error) {
|
|
560
|
-
const activePlan = resolveActiveExecutionPlan(state);
|
|
561
|
-
const wrapped = new Error(
|
|
562
|
-
`[Pipeline] ${contextLabel}: finiteness guard triggered for kernelPath ` +
|
|
563
|
-
`"${activePlan.kernelPathId ?? 'none'}" under fail-fast policy. ` +
|
|
564
|
-
'Resolve the unstable path with an explicit capability-aware execution override, ' +
|
|
565
|
-
'or opt into alternate-plan recovery with ' +
|
|
566
|
-
'runtime.inference.compute.rangeAwareSelectiveWidening.onTrigger="fallback-plan".',
|
|
567
|
-
error instanceof Error ? { cause: error } : undefined
|
|
568
|
-
);
|
|
569
|
-
wrapped.name = error?.name === 'FinitenessError' ? error.name : 'FinitenessError';
|
|
570
|
-
return wrapped;
|
|
571
|
-
}
|
|
572
|
-
|
|
573
|
-
function assertResolvedKVDtype(kvDtype, contextLabel) {
|
|
574
|
-
if (kvDtype === 'f16' || kvDtype === 'f32') {
|
|
575
|
-
return kvDtype;
|
|
576
|
-
}
|
|
577
|
-
throw new Error(
|
|
578
|
-
`[Pipeline] ${contextLabel}: expected execution-plan kvDtype to resolve to "f16" or "f32", ` +
|
|
579
|
-
`got ${JSON.stringify(kvDtype)}.`
|
|
580
|
-
);
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
function resolveTargetPlanKVDtype(plan, contextLabel) {
|
|
584
|
-
return assertResolvedKVDtype(
|
|
585
|
-
plan?.kernelPath?.kvDtype ?? plan?.activationDtype ?? null,
|
|
586
|
-
contextLabel
|
|
587
|
-
);
|
|
588
|
-
}
|
|
589
|
-
|
|
590
|
-
function resolveCurrentKVCacheDtype(state, plan, contextLabel) {
|
|
591
|
-
return assertResolvedKVDtype(
|
|
592
|
-
state?.kvCache?.kvDtype
|
|
593
|
-
?? plan?.kernelPath?.kvDtype
|
|
594
|
-
?? state?.runtimeConfig?.inference?.session?.kvcache?.kvDtype
|
|
595
|
-
?? plan?.activationDtype
|
|
596
|
-
?? null,
|
|
597
|
-
contextLabel
|
|
598
|
-
);
|
|
599
|
-
}
|
|
600
|
-
|
|
601
|
-
function cloneRuntimeInferenceWithKVDtype(state, kvDtype) {
|
|
602
|
-
const runtimeInference = state?.runtimeConfig?.inference;
|
|
603
|
-
if (!runtimeInference?.session?.kvcache) {
|
|
604
|
-
throw new Error(
|
|
605
|
-
'[Pipeline] runtime.inference.session.kvcache is required for finiteness fallback KV-cache recovery.'
|
|
606
|
-
);
|
|
607
|
-
}
|
|
608
|
-
return mergeRuntimeValues(runtimeInference, {
|
|
609
|
-
session: {
|
|
610
|
-
kvcache: {
|
|
611
|
-
kvDtype,
|
|
612
|
-
},
|
|
613
|
-
},
|
|
614
|
-
});
|
|
615
|
-
}
|
|
616
121
|
|
|
617
122
|
export class PipelineGenerator {
|
|
618
123
|
|
|
@@ -812,14 +317,36 @@ export class PipelineGenerator {
|
|
|
812
317
|
_sampleNextTokenFromLogits(logits, generatedIds, opts) {
|
|
813
318
|
const sampledLogits = Float32Array.from(logits);
|
|
814
319
|
applyRepetitionPenalty(sampledLogits, generatedIds, opts.repetitionPenalty);
|
|
320
|
+
// Optional pre-sample logit mask. Callers pass `opts.logitMaskFn` to
|
|
321
|
+
// implement grammar/schema-constrained decoding. The hook receives the
|
|
322
|
+
// mutable logit buffer (after repetition penalty) plus the running token
|
|
323
|
+
// sequence so it can track parse state across decode steps.
|
|
324
|
+
if (typeof opts?.logitMaskFn === "function") {
|
|
325
|
+
try {
|
|
326
|
+
opts.logitMaskFn(sampledLogits, {
|
|
327
|
+
generatedIds,
|
|
328
|
+
tokenizer: this.#state.tokenizer ?? null,
|
|
329
|
+
vocabSize: this.#state.modelConfig?.vocabSize ?? sampledLogits.length,
|
|
330
|
+
});
|
|
331
|
+
} catch (maskError) {
|
|
332
|
+
log.warn("Pipeline", `logitMaskFn threw; continuing without mask: ${maskError}`);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
815
335
|
const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
|
|
816
|
-
|
|
336
|
+
const tokenId = sample(sampledLogits, {
|
|
817
337
|
temperature: opts.temperature,
|
|
818
338
|
topP: opts.topP,
|
|
819
339
|
topK: opts.topK,
|
|
820
340
|
padTokenId,
|
|
821
341
|
seed: opts.seed,
|
|
822
342
|
});
|
|
343
|
+
if (typeof opts.onLogits === 'function') {
|
|
344
|
+
opts.onLogits(sampledLogits, {
|
|
345
|
+
tokenId,
|
|
346
|
+
inputTokenCount: Array.isArray(generatedIds) ? generatedIds.length : null,
|
|
347
|
+
});
|
|
348
|
+
}
|
|
349
|
+
return tokenId;
|
|
823
350
|
}
|
|
824
351
|
|
|
825
352
|
async _prefillPromptToLogits(prompt, opts, contextLabel) {
|
|
@@ -880,6 +407,11 @@ export class PipelineGenerator {
|
|
|
880
407
|
return this._matchesStopSequence(generatedIds, runtime.stopSequenceStart, opts.stopSequences);
|
|
881
408
|
}
|
|
882
409
|
|
|
410
|
+
_recordStopReason(reason, tokenId = null) {
|
|
411
|
+
this.#state.stats.stopReason = reason;
|
|
412
|
+
this.#state.stats.stopTokenId = Number.isInteger(tokenId) ? tokenId : null;
|
|
413
|
+
}
|
|
414
|
+
|
|
883
415
|
async *_generateTokensInternal(prompt, options = {}, mode = 'text') {
|
|
884
416
|
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
885
417
|
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
@@ -893,6 +425,8 @@ export class PipelineGenerator {
|
|
|
893
425
|
this.#state.stats.decodeRecordMs = 0;
|
|
894
426
|
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
895
427
|
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
428
|
+
this.#state.stats.prefillRecordMs = 0;
|
|
429
|
+
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
896
430
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
897
431
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
898
432
|
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
@@ -904,10 +438,14 @@ export class PipelineGenerator {
|
|
|
904
438
|
this.#state.stats.plePreparedTokenCacheBytes = 0;
|
|
905
439
|
this.#state.stats.decodeMode = null;
|
|
906
440
|
this.#state.stats.batchGuardReason = null;
|
|
441
|
+
this.#state.stats.stopReason = null;
|
|
442
|
+
this.#state.stats.stopTokenId = null;
|
|
907
443
|
this.#state.stats.ttftMs = 0;
|
|
908
444
|
const startTime = performance.now();
|
|
909
445
|
|
|
910
446
|
const opts = resolveGenerateOptions(this.#state, options);
|
|
447
|
+
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
448
|
+
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
911
449
|
// Validate and normalize sampling parameters through single source of truth
|
|
912
450
|
const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
|
|
913
451
|
opts.temperature = samplingConfig.temperature;
|
|
@@ -916,7 +454,8 @@ export class PipelineGenerator {
|
|
|
916
454
|
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
917
455
|
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
918
456
|
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
919
|
-
|
|
457
|
+
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
458
|
+
if (diagnosticsEnabled || tsirFixtureCfg) {
|
|
920
459
|
const captureConfig = {
|
|
921
460
|
...createDefaultCaptureConfig(),
|
|
922
461
|
enabled: true,
|
|
@@ -925,13 +464,20 @@ export class PipelineGenerator {
|
|
|
925
464
|
};
|
|
926
465
|
validateCaptureConfig(captureConfig);
|
|
927
466
|
this.#state.operatorDiagnostics = {
|
|
928
|
-
enabled: true,
|
|
467
|
+
enabled: diagnosticsEnabled === true,
|
|
929
468
|
captureConfig,
|
|
930
|
-
emitter: new OperatorEventEmitter({
|
|
469
|
+
emitter: diagnosticsEnabled ? new OperatorEventEmitter({
|
|
931
470
|
modelHash: this.#state.manifest?.modelId ?? null,
|
|
932
471
|
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
933
472
|
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
934
|
-
}),
|
|
473
|
+
}) : null,
|
|
474
|
+
tsirFixture: tsirFixtureCfg ? {
|
|
475
|
+
dir: tsirFixtureCfg.dir,
|
|
476
|
+
layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
|
|
477
|
+
? tsirFixtureCfg.layerFilter
|
|
478
|
+
: null,
|
|
479
|
+
records: [],
|
|
480
|
+
} : null,
|
|
935
481
|
};
|
|
936
482
|
}
|
|
937
483
|
const activePlan = opts.executionPlan ?? resolveActiveExecutionPlan(this.#state);
|
|
@@ -1051,6 +597,7 @@ export class PipelineGenerator {
|
|
|
1051
597
|
yield* emitToken(this, firstToken, decodeToken);
|
|
1052
598
|
|
|
1053
599
|
if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
|
|
600
|
+
this._recordStopReason('stop-token-or-sequence', firstToken);
|
|
1054
601
|
this.#state.stats.decodeTimeMs = 0;
|
|
1055
602
|
this.#state.stats.tokensGenerated = 1;
|
|
1056
603
|
this.#state.stats.decodeTokens = 1;
|
|
@@ -1277,6 +824,31 @@ export class PipelineGenerator {
|
|
|
1277
824
|
// Generation Public API
|
|
1278
825
|
// ==========================================================================
|
|
1279
826
|
|
|
827
|
+
/**
|
|
828
|
+
* Truncate the KV cache back to `seqLen` tokens and set `currentSeqLen` to
|
|
829
|
+
* match. Intended for "prefix-reuse" workflows where a caller wants to run
|
|
830
|
+
* several decodes that share a common prompt prefix: prefill once with the
|
|
831
|
+
* shared prefix, decode the first tail, then `resetToSeqLen(prefixLen)` to
|
|
832
|
+
* drop the tail's KV entries and reuse the prefix KV for the next tail.
|
|
833
|
+
*
|
|
834
|
+
* Only valid when no decode is in progress.
|
|
835
|
+
*/
|
|
836
|
+
resetToSeqLen(seqLen) {
|
|
837
|
+
if (this.#state.isGenerating) {
|
|
838
|
+
throw new Error('InferencePipeline.resetToSeqLen: cannot reset while generation is in progress');
|
|
839
|
+
}
|
|
840
|
+
const target = Math.max(0, Math.floor(Number(seqLen) || 0));
|
|
841
|
+
if (!Number.isFinite(target)) {
|
|
842
|
+
throw new Error('InferencePipeline.resetToSeqLen: seqLen must be a finite non-negative integer');
|
|
843
|
+
}
|
|
844
|
+
if (target > this.#state.currentSeqLen) {
|
|
845
|
+
throw new Error(
|
|
846
|
+
`InferencePipeline.resetToSeqLen: target ${target} exceeds currentSeqLen ${this.#state.currentSeqLen}`
|
|
847
|
+
);
|
|
848
|
+
}
|
|
849
|
+
this.#state.kvCache?.truncate?.(target);
|
|
850
|
+
this.#state.currentSeqLen = target;
|
|
851
|
+
}
|
|
1280
852
|
|
|
1281
853
|
async *generate(prompt, options = {}) {
|
|
1282
854
|
yield* this._generateTokensInternal(prompt, options, 'text');
|
|
@@ -1299,6 +871,8 @@ export class PipelineGenerator {
|
|
|
1299
871
|
this.#state.stats.decodeRecordMs = 0;
|
|
1300
872
|
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
1301
873
|
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
874
|
+
this.#state.stats.prefillRecordMs = 0;
|
|
875
|
+
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
1302
876
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
1303
877
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
1304
878
|
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
@@ -1310,9 +884,12 @@ export class PipelineGenerator {
|
|
|
1310
884
|
this.#state.stats.plePreparedTokenCacheBytes = 0;
|
|
1311
885
|
this.#state.stats.decodeMode = null;
|
|
1312
886
|
this.#state.stats.batchGuardReason = null;
|
|
887
|
+
this.#state.stats.stopReason = null;
|
|
888
|
+
this.#state.stats.stopTokenId = null;
|
|
1313
889
|
this.#state.stats.ttftMs = 0;
|
|
1314
890
|
const startTime = performance.now();
|
|
1315
891
|
const opts = resolveGenerateOptions(this.#state, options);
|
|
892
|
+
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
1316
893
|
// Validate and normalize sampling parameters through single source of truth
|
|
1317
894
|
const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
|
|
1318
895
|
opts.temperature = samplingConfig.temperature;
|
|
@@ -1321,7 +898,8 @@ export class PipelineGenerator {
|
|
|
1321
898
|
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
1322
899
|
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
1323
900
|
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
1324
|
-
|
|
901
|
+
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
902
|
+
if (diagnosticsEnabled || tsirFixtureCfg) {
|
|
1325
903
|
const captureConfig = {
|
|
1326
904
|
...createDefaultCaptureConfig(),
|
|
1327
905
|
enabled: true,
|
|
@@ -1330,13 +908,20 @@ export class PipelineGenerator {
|
|
|
1330
908
|
};
|
|
1331
909
|
validateCaptureConfig(captureConfig);
|
|
1332
910
|
this.#state.operatorDiagnostics = {
|
|
1333
|
-
enabled: true,
|
|
911
|
+
enabled: diagnosticsEnabled === true,
|
|
1334
912
|
captureConfig,
|
|
1335
|
-
emitter: new OperatorEventEmitter({
|
|
913
|
+
emitter: diagnosticsEnabled ? new OperatorEventEmitter({
|
|
1336
914
|
modelHash: this.#state.manifest?.modelId ?? null,
|
|
1337
915
|
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
1338
916
|
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
1339
|
-
}),
|
|
917
|
+
}) : null,
|
|
918
|
+
tsirFixture: tsirFixtureCfg ? {
|
|
919
|
+
dir: tsirFixtureCfg.dir,
|
|
920
|
+
layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
|
|
921
|
+
? tsirFixtureCfg.layerFilter
|
|
922
|
+
: null,
|
|
923
|
+
records: [],
|
|
924
|
+
} : null,
|
|
1340
925
|
};
|
|
1341
926
|
}
|
|
1342
927
|
|
|
@@ -1388,6 +973,7 @@ export class PipelineGenerator {
|
|
|
1388
973
|
tokenIds.push(tokenId);
|
|
1389
974
|
}
|
|
1390
975
|
} else {
|
|
976
|
+
this._recordStopReason('stop-token-or-sequence', firstToken);
|
|
1391
977
|
this.#state.stats.decodeTimeMs = 0;
|
|
1392
978
|
this.#state.stats.tokensGenerated = 1;
|
|
1393
979
|
this.#state.stats.decodeTokens = 1;
|
|
@@ -1726,6 +1312,7 @@ export class PipelineGenerator {
|
|
|
1726
1312
|
if (options.onToken) options.onToken(firstToken, firstText);
|
|
1727
1313
|
|
|
1728
1314
|
if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
|
|
1315
|
+
this._recordStopReason('stop-token-or-sequence', firstToken);
|
|
1729
1316
|
this.#state.stats.decodeTimeMs = 0;
|
|
1730
1317
|
this.#state.stats.tokensGenerated = 1;
|
|
1731
1318
|
this.#state.stats.decodeTokens = 1;
|
|
@@ -1852,7 +1439,10 @@ export class PipelineGenerator {
|
|
|
1852
1439
|
}
|
|
1853
1440
|
|
|
1854
1441
|
while (tokensGenerated < opts.maxTokens) {
|
|
1855
|
-
if (options.signal?.aborted)
|
|
1442
|
+
if (options.signal?.aborted) {
|
|
1443
|
+
this._recordStopReason('aborted');
|
|
1444
|
+
break;
|
|
1445
|
+
}
|
|
1856
1446
|
if (this._hasFinitenessFallbackWindow() && useBatchPath) {
|
|
1857
1447
|
useBatchPath = false;
|
|
1858
1448
|
}
|
|
@@ -1901,9 +1491,15 @@ export class PipelineGenerator {
|
|
|
1901
1491
|
if (batchTokens.length > 0 && options.onBatch) options.onBatch(batchTokens);
|
|
1902
1492
|
if (opts.stopSequences.length > 0) {
|
|
1903
1493
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
1904
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq)))
|
|
1494
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
1495
|
+
this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
|
|
1496
|
+
break;
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
if (hitStop) {
|
|
1500
|
+
this._recordStopReason('stop-token');
|
|
1501
|
+
break;
|
|
1905
1502
|
}
|
|
1906
|
-
if (hitStop) break;
|
|
1907
1503
|
if (shouldDisableBatchDecodeAfterShortBatch({
|
|
1908
1504
|
hitStop,
|
|
1909
1505
|
actualCount: batchResult.actualCount,
|
|
@@ -1941,7 +1537,10 @@ export class PipelineGenerator {
|
|
|
1941
1537
|
if (options.onToken) options.onToken(nextToken, tokenText);
|
|
1942
1538
|
}
|
|
1943
1539
|
this._consumeFinitenessFallbackToken(opts);
|
|
1944
|
-
if (isStopToken(nextToken, stopTokenIds, eosToken))
|
|
1540
|
+
if (isStopToken(nextToken, stopTokenIds, eosToken)) {
|
|
1541
|
+
this._recordStopReason('stop-token', nextToken);
|
|
1542
|
+
break;
|
|
1543
|
+
}
|
|
1945
1544
|
}
|
|
1946
1545
|
} else if (opts.speculation?.mode === 'self') {
|
|
1947
1546
|
// Self-speculation: decode one base token plus a configurable burst of
|
|
@@ -1979,11 +1578,17 @@ export class PipelineGenerator {
|
|
|
1979
1578
|
}
|
|
1980
1579
|
this._consumeFinitenessFallbackToken(opts);
|
|
1981
1580
|
|
|
1982
|
-
if (isStopToken(baseToken, stopTokenIds, eosToken))
|
|
1581
|
+
if (isStopToken(baseToken, stopTokenIds, eosToken)) {
|
|
1582
|
+
this._recordStopReason('stop-token', baseToken);
|
|
1583
|
+
break;
|
|
1584
|
+
}
|
|
1983
1585
|
if (tokensGenerated >= opts.maxTokens) break;
|
|
1984
1586
|
if (opts.stopSequences.length > 0) {
|
|
1985
1587
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
1986
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq)))
|
|
1588
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
1589
|
+
this._recordStopReason('stop-sequence', baseToken);
|
|
1590
|
+
break;
|
|
1591
|
+
}
|
|
1987
1592
|
}
|
|
1988
1593
|
|
|
1989
1594
|
for (let specIndex = 0; specIndex < speculativeBurstTokens; specIndex += 1) {
|
|
@@ -2022,19 +1627,27 @@ export class PipelineGenerator {
|
|
|
2022
1627
|
}
|
|
2023
1628
|
|
|
2024
1629
|
if (isStopToken(specToken, stopTokenIds, eosToken)) {
|
|
1630
|
+
this._recordStopReason('stop-token', specToken);
|
|
2025
1631
|
break;
|
|
2026
1632
|
}
|
|
2027
1633
|
if (opts.stopSequences.length > 0) {
|
|
2028
1634
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
2029
1635
|
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
1636
|
+
this._recordStopReason('stop-sequence', specToken);
|
|
2030
1637
|
break;
|
|
2031
1638
|
}
|
|
2032
1639
|
}
|
|
2033
1640
|
}
|
|
2034
|
-
if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken))
|
|
1641
|
+
if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken)) {
|
|
1642
|
+
this._recordStopReason('stop-token', generatedIds[generatedIds.length - 1]);
|
|
1643
|
+
break;
|
|
1644
|
+
}
|
|
2035
1645
|
if (opts.stopSequences.length > 0) {
|
|
2036
1646
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
2037
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq)))
|
|
1647
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
1648
|
+
this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
|
|
1649
|
+
break;
|
|
1650
|
+
}
|
|
2038
1651
|
}
|
|
2039
1652
|
} else {
|
|
2040
1653
|
const tokenStart = performance.now();
|
|
@@ -2088,14 +1701,23 @@ export class PipelineGenerator {
|
|
|
2088
1701
|
log.debug('Decode', `#${tokensGenerated} "${tokenText}" ${tokenTime.toFixed(0)}ms (${tokPerSec.toFixed(2)} tok/s avg)`);
|
|
2089
1702
|
}
|
|
2090
1703
|
|
|
2091
|
-
if (isStopToken(nextToken, stopTokenIds, eosToken))
|
|
1704
|
+
if (isStopToken(nextToken, stopTokenIds, eosToken)) {
|
|
1705
|
+
this._recordStopReason('stop-token', nextToken);
|
|
1706
|
+
break;
|
|
1707
|
+
}
|
|
2092
1708
|
if (opts.stopSequences.length > 0) {
|
|
2093
1709
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
2094
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq)))
|
|
1710
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
1711
|
+
this._recordStopReason('stop-sequence', nextToken);
|
|
1712
|
+
break;
|
|
1713
|
+
}
|
|
2095
1714
|
}
|
|
2096
1715
|
}
|
|
2097
1716
|
}
|
|
2098
1717
|
|
|
1718
|
+
if (!this.#state.stats.stopReason) {
|
|
1719
|
+
this._recordStopReason(tokensGenerated >= opts.maxTokens ? 'max-tokens' : 'completed');
|
|
1720
|
+
}
|
|
2099
1721
|
this.#state.stats.decodeTimeMs = performance.now() - decodeStart;
|
|
2100
1722
|
this.#state.stats.tokensGenerated = tokensGenerated;
|
|
2101
1723
|
this.#state.stats.decodeTokens = tokensGenerated;
|
|
@@ -2321,16 +1943,35 @@ export class PipelineGenerator {
|
|
|
2321
1943
|
// Chunked recorder submission: submit every N layers to release tracked intermediate
|
|
2322
1944
|
// buffers, preventing unbounded memory growth during large prefills. Critical for
|
|
2323
1945
|
// replay_prefill models where each decode step re-runs a prefill-style layer pass.
|
|
2324
|
-
const
|
|
1946
|
+
const prefillRecorderChunkLayers = resolvePrefillRecorderChunkLayers({
|
|
1947
|
+
hasGpuSplitPerLayerInputs: context.perLayerInputsSession?.materialization === 'gpu_split_tables',
|
|
1948
|
+
numTokens,
|
|
1949
|
+
});
|
|
2325
1950
|
|
|
2326
1951
|
let currentHiddenBuffer = hiddenStates.buffer;
|
|
1952
|
+
let prefillRecordMs = 0;
|
|
1953
|
+
let prefillSubmitWaitMs = 0;
|
|
2327
1954
|
try {
|
|
2328
1955
|
for (let l = 0; l < config.numLayers; l++) {
|
|
1956
|
+
// Per-layer hard cancellation: when the caller's AbortSignal aborts,
|
|
1957
|
+
// exit the prefill loop between layer dispatches rather than
|
|
1958
|
+
// continuing to burn GPU on superseded work. Granularity is one
|
|
1959
|
+
// layer (~5-50ms), which is the fastest cancel granularity WebGPU
|
|
1960
|
+
// exposes today.
|
|
1961
|
+
if (opts?.signal?.aborted) {
|
|
1962
|
+
const reason = typeof opts.signal.reason === "string" ? opts.signal.reason : "Doppler: prefill aborted";
|
|
1963
|
+
const err = new Error(reason);
|
|
1964
|
+
err.name = "AbortError";
|
|
1965
|
+
err.code = "ABORT_ERR";
|
|
1966
|
+
throw err;
|
|
1967
|
+
}
|
|
2329
1968
|
context.recorder = currentRecorder;
|
|
2330
1969
|
context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
|
|
2331
1970
|
|
|
2332
1971
|
const prevBuffer = currentHiddenBuffer;
|
|
1972
|
+
const layerRecordStart = performance.now();
|
|
2333
1973
|
const layerOutput = await processLayer(l, currentHiddenBuffer, numTokens, true, context);
|
|
1974
|
+
prefillRecordMs += performance.now() - layerRecordStart;
|
|
2334
1975
|
if (!isGpuBufferInstance(layerOutput)) throw new Error('Expected GPUBuffer from processLayer');
|
|
2335
1976
|
currentHiddenBuffer = layerOutput;
|
|
2336
1977
|
releasePerLayerInputBuffer(
|
|
@@ -2348,7 +1989,7 @@ export class PipelineGenerator {
|
|
|
2348
1989
|
const isChunkBoundary = !isCheckpoint
|
|
2349
1990
|
&& currentRecorder
|
|
2350
1991
|
&& l < config.numLayers - 1
|
|
2351
|
-
&& (l + 1) %
|
|
1992
|
+
&& (l + 1) % prefillRecorderChunkLayers === 0;
|
|
2352
1993
|
|
|
2353
1994
|
if (isCheckpoint && currentRecorder) {
|
|
2354
1995
|
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
@@ -2408,8 +2049,25 @@ export class PipelineGenerator {
|
|
|
2408
2049
|
currentRecorder,
|
|
2409
2050
|
'prefill_chunk_carry'
|
|
2410
2051
|
);
|
|
2411
|
-
|
|
2412
|
-
|
|
2052
|
+
// Chunk boundary exists only to bound intermediate buffer lifetime.
|
|
2053
|
+
// When the runtime opts into async chunk submit AND profile timings
|
|
2054
|
+
// are not being collected, skip the CPU-GPU wait: queue order is
|
|
2055
|
+
// preserved across submits and deferred cleanup still releases
|
|
2056
|
+
// tracked buffers when GPU work completes. Profile runs keep the
|
|
2057
|
+
// sync path because resolveProfileTimings requires the submitted
|
|
2058
|
+
// work to be done.
|
|
2059
|
+
const manifestSubmit = this.#state.modelConfig?.sessionSettings?.prefillChunkSubmitMode;
|
|
2060
|
+
const chunkSubmitMode = (manifestSubmit !== null && manifestSubmit !== undefined)
|
|
2061
|
+
? manifestSubmit
|
|
2062
|
+
: (this.#state.runtimeConfig?.inference?.session?.prefillChunkSubmitMode ?? 'sync');
|
|
2063
|
+
const chunkSubmitStart = performance.now();
|
|
2064
|
+
if (chunkSubmitMode === 'async' && !opts.profile) {
|
|
2065
|
+
currentRecorder.submit({ cleanup: 'deferred' });
|
|
2066
|
+
} else {
|
|
2067
|
+
await currentRecorder.submitAndWait();
|
|
2068
|
+
await recordProfile(currentRecorder);
|
|
2069
|
+
}
|
|
2070
|
+
prefillSubmitWaitMs += performance.now() - chunkSubmitStart;
|
|
2413
2071
|
currentRecorder = createRecorder('prefill-chunk');
|
|
2414
2072
|
}
|
|
2415
2073
|
}
|
|
@@ -2426,8 +2084,26 @@ export class PipelineGenerator {
|
|
|
2426
2084
|
}
|
|
2427
2085
|
}
|
|
2428
2086
|
releaseSharedAttentionState(context.sharedAttentionState, currentRecorder);
|
|
2087
|
+
this.#state.stats.prefillRecordMs = (this.#state.stats.prefillRecordMs ?? 0) + prefillRecordMs;
|
|
2088
|
+
this.#state.stats.prefillSubmitWaitMs = (this.#state.stats.prefillSubmitWaitMs ?? 0) + prefillSubmitWaitMs;
|
|
2429
2089
|
}
|
|
2430
2090
|
|
|
2091
|
+
{
|
|
2092
|
+
const tsirFixtureCfg = this.#state.operatorDiagnostics?.tsirFixture ?? null;
|
|
2093
|
+
if (tsirFixtureCfg && Array.isArray(tsirFixtureCfg.pendingReads) && tsirFixtureCfg.pendingReads.length > 0) {
|
|
2094
|
+
if (currentRecorder) {
|
|
2095
|
+
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
2096
|
+
currentHiddenBuffer,
|
|
2097
|
+
currentRecorder,
|
|
2098
|
+
'prefill_tsir_drain_carry'
|
|
2099
|
+
);
|
|
2100
|
+
await currentRecorder.submitAndWait();
|
|
2101
|
+
await recordProfile(currentRecorder);
|
|
2102
|
+
currentRecorder = undefined;
|
|
2103
|
+
}
|
|
2104
|
+
await drainPendingTsirReads(tsirFixtureCfg);
|
|
2105
|
+
}
|
|
2106
|
+
}
|
|
2431
2107
|
if (this.#state.finitenessBuffer) {
|
|
2432
2108
|
if (currentRecorder) {
|
|
2433
2109
|
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|