doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -14,15 +14,11 @@ import {
|
|
|
14
14
|
validateCaptureConfig,
|
|
15
15
|
} from '../../../debug/index.js';
|
|
16
16
|
import { validateCallTimeOptions } from '../../../config/param-validator.js';
|
|
17
|
-
import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
|
|
18
17
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
19
|
-
import { castF32ToF16 } from '../../../gpu/kernels/cast.js';
|
|
20
|
-
import { f32ToF16Array } from '../../kv-cache/types.js';
|
|
21
18
|
|
|
22
19
|
// Pipeline sub-modules
|
|
23
20
|
import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './sampling.js';
|
|
24
|
-
import {
|
|
25
|
-
import { formatChatMessages } from './chat-format.js';
|
|
21
|
+
import { createKVCache, isStopToken } from './init.js';
|
|
26
22
|
import { embed } from './embed.js';
|
|
27
23
|
import { processLayer } from './layer.js';
|
|
28
24
|
import { computeLogits, recordLogitsGPU, extractLastPositionLogits, applySoftcapping } from './logits/index.js';
|
|
@@ -43,6 +39,7 @@ import {
|
|
|
43
39
|
debugCheckBuffer as debugCheckBufferHelper,
|
|
44
40
|
getLogitsConfig,
|
|
45
41
|
getLogitsWeights,
|
|
42
|
+
resolvePerLayerInputsSession,
|
|
46
43
|
releaseSharedAttentionState,
|
|
47
44
|
} from './generator-helpers.js';
|
|
48
45
|
import {
|
|
@@ -61,12 +58,14 @@ import { resolveSamplingConfig } from './sampling-config.js';
|
|
|
61
58
|
import { decodeReadback, getLogitsHealth } from './debug-utils/index.js';
|
|
62
59
|
import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
|
|
63
60
|
import { resolveDeferredRoundingWindowTokens } from './finiteness-policy.js';
|
|
61
|
+
import { drainPendingTsirReads } from './tsir-fixture-writer.js';
|
|
64
62
|
import {
|
|
65
63
|
activateFallbackExecutionPlan,
|
|
66
64
|
hasFallbackExecutionPlan,
|
|
67
65
|
rebaseExecutionSessionPlan,
|
|
68
66
|
resetActiveExecutionPlan,
|
|
69
67
|
resolveMaxBatchDecodeTokens,
|
|
68
|
+
resolvePrefillRecorderChunkLayers,
|
|
70
69
|
resolveActiveExecutionPlan,
|
|
71
70
|
setActiveExecutionPlan,
|
|
72
71
|
} from './execution-plan.js';
|
|
@@ -80,15 +79,37 @@ import {
|
|
|
80
79
|
preparePerLayerInputs,
|
|
81
80
|
createPleBufferCache,
|
|
82
81
|
prefetchPerLayerRow,
|
|
83
|
-
ensurePleScaledProjectionNormWeight,
|
|
84
|
-
ensurePleGpuHotVocabularyRuntime,
|
|
85
|
-
ensurePleGpuSplitTablesRuntime,
|
|
86
82
|
getPleHotVocabularyRuntime,
|
|
87
83
|
hasGpuSplitPerLayerInputEmbeddings,
|
|
88
84
|
hasRangeBackedPerLayerInputEmbeddings,
|
|
89
85
|
} from './per-layer-inputs.js';
|
|
90
86
|
import { createTensor } from '../../../gpu/tensor.js';
|
|
91
|
-
|
|
87
|
+
|
|
88
|
+
// Extracted standalone helpers
|
|
89
|
+
import {
|
|
90
|
+
resolvePromptInput,
|
|
91
|
+
releasePerLayerInputBuffer,
|
|
92
|
+
normalizePrefixEmbeddingOverride,
|
|
93
|
+
resolvePrefillEmbeddingInputIds,
|
|
94
|
+
resolvePrefillMultimodalBidirectionalSpan,
|
|
95
|
+
applyPrefixEmbeddingOverride,
|
|
96
|
+
shouldDisablePrefillCommandBatching,
|
|
97
|
+
} from './generator-prefill-helpers.js';
|
|
98
|
+
import {
|
|
99
|
+
shouldDisableBatchDecodeAfterShortBatch,
|
|
100
|
+
resolveHotVocabularyBatchDecodeAvailability,
|
|
101
|
+
primePleDecodeRuntimeCache,
|
|
102
|
+
recordPrefillProfileStep,
|
|
103
|
+
resolveTokenText,
|
|
104
|
+
usesReplayPrefillDecode,
|
|
105
|
+
assertIncrementalDecodeSupport,
|
|
106
|
+
summarizeExecutionPlan,
|
|
107
|
+
shouldRetryWithFinitenessFallback,
|
|
108
|
+
createUnhandledFinitenessPolicyError,
|
|
109
|
+
resolveTargetPlanKVDtype,
|
|
110
|
+
resolveCurrentKVCacheDtype,
|
|
111
|
+
cloneRuntimeInferenceWithKVDtype,
|
|
112
|
+
} from './generator-decode-policy.js';
|
|
92
113
|
|
|
93
114
|
let intentBundleModulePromise = null;
|
|
94
115
|
|
|
@@ -97,507 +118,6 @@ async function getExperimentalIntentBundleModule() {
|
|
|
97
118
|
return intentBundleModulePromise;
|
|
98
119
|
}
|
|
99
120
|
|
|
100
|
-
function isStructuredChatRequest(prompt) {
|
|
101
|
-
return prompt != null
|
|
102
|
-
&& typeof prompt === 'object'
|
|
103
|
-
&& !Array.isArray(prompt)
|
|
104
|
-
&& Array.isArray(prompt.messages);
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
export function shouldDisableBatchDecodeAfterShortBatch({ hitStop, actualCount, requestedCount }) {
|
|
108
|
-
return hitStop !== true
|
|
109
|
-
&& Number.isInteger(actualCount)
|
|
110
|
-
&& Number.isInteger(requestedCount)
|
|
111
|
-
&& actualCount > 0
|
|
112
|
-
&& actualCount < requestedCount;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
export function resolveHotVocabularyBatchDecodeAvailability({
|
|
116
|
-
hasRangeBackedPerLayerInputs,
|
|
117
|
-
pleHotVocabularyRuntime,
|
|
118
|
-
tokenId,
|
|
119
|
-
}) {
|
|
120
|
-
if (hasRangeBackedPerLayerInputs !== true) {
|
|
121
|
-
return false;
|
|
122
|
-
}
|
|
123
|
-
if (!pleHotVocabularyRuntime || typeof pleHotVocabularyRuntime !== 'object') {
|
|
124
|
-
return false;
|
|
125
|
-
}
|
|
126
|
-
const sentinelIndex = pleHotVocabularyRuntime.sentinelIndex;
|
|
127
|
-
if (!Number.isInteger(sentinelIndex)) {
|
|
128
|
-
return false;
|
|
129
|
-
}
|
|
130
|
-
if (!Number.isInteger(tokenId)) {
|
|
131
|
-
return false;
|
|
132
|
-
}
|
|
133
|
-
const hotTokenIndex = pleHotVocabularyRuntime.hotTokenIndexMap?.[tokenId] ?? sentinelIndex;
|
|
134
|
-
return hotTokenIndex !== sentinelIndex;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
|
|
138
|
-
await ensurePleGpuHotVocabularyRuntime({
|
|
139
|
-
config: state.modelConfig,
|
|
140
|
-
weights: state.weights,
|
|
141
|
-
perLayerInputsSession: state.runtimeConfig.inference.session?.perLayerInputs ?? state.modelConfig?.perLayerInputsSession ?? null,
|
|
142
|
-
debugFlags: state.debugFlags,
|
|
143
|
-
tokenizer: state.tokenizer ?? null,
|
|
144
|
-
seedTokenIds: Array.isArray(seedTokenIds) ? seedTokenIds : null,
|
|
145
|
-
});
|
|
146
|
-
await ensurePleGpuSplitTablesRuntime({
|
|
147
|
-
config: state.modelConfig,
|
|
148
|
-
weights: state.weights,
|
|
149
|
-
perLayerInputsSession: state.runtimeConfig.inference.session?.perLayerInputs ?? state.modelConfig?.perLayerInputsSession ?? null,
|
|
150
|
-
debugFlags: state.debugFlags,
|
|
151
|
-
});
|
|
152
|
-
await ensurePleScaledProjectionNormWeight({
|
|
153
|
-
config: state.modelConfig,
|
|
154
|
-
weights: state.weights,
|
|
155
|
-
weightConfig: {
|
|
156
|
-
rmsNormWeightOffset: state.modelConfig?.rmsNormWeightOffset ?? false,
|
|
157
|
-
},
|
|
158
|
-
debugFlags: state.debugFlags,
|
|
159
|
-
});
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
function recordPrefillProfileStep(state, entry) {
|
|
163
|
-
if (!entry?.timings || Object.keys(entry.timings).length === 0) return;
|
|
164
|
-
if (!state.stats.prefillProfileSteps) {
|
|
165
|
-
state.stats.prefillProfileSteps = [];
|
|
166
|
-
}
|
|
167
|
-
state.stats.prefillProfileSteps.push(entry);
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
|
|
171
|
-
const chatOptions = state.modelConfig.chatTemplateThinking === true ? { thinking: true } : undefined;
|
|
172
|
-
if (typeof prompt === 'string') {
|
|
173
|
-
if (useChatTemplate && state.modelConfig.chatTemplateType) {
|
|
174
|
-
if (state.modelConfig.chatTemplateType === 'translategemma') {
|
|
175
|
-
throw new Error(
|
|
176
|
-
`[Pipeline] ${contextLabel}: translategemma chat template requires structured messages. ` +
|
|
177
|
-
'Pass { messages: [...] } instead of a plain string prompt.'
|
|
178
|
-
);
|
|
179
|
-
}
|
|
180
|
-
return applyChatTemplate(prompt, state.modelConfig.chatTemplateType, chatOptions);
|
|
181
|
-
}
|
|
182
|
-
return prompt;
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
if (prompt != null && typeof prompt === 'object' && !Array.isArray(prompt) && 'messages' in prompt && !Array.isArray(prompt.messages)) {
|
|
186
|
-
throw new Error(
|
|
187
|
-
`[Pipeline] ${contextLabel}: prompt.messages must be an array of chat messages, got ${typeof prompt.messages}. ` +
|
|
188
|
-
'Pass { messages: [{ role: "user", content: "..." }, ...] }.'
|
|
189
|
-
);
|
|
190
|
-
}
|
|
191
|
-
const messages = isStructuredChatRequest(prompt)
|
|
192
|
-
? prompt.messages
|
|
193
|
-
: (Array.isArray(prompt) ? prompt : null);
|
|
194
|
-
if (!messages) {
|
|
195
|
-
throw new Error(
|
|
196
|
-
`[Pipeline] ${contextLabel}: prompt must be a string, chat message array, or { messages: [...] }.`
|
|
197
|
-
);
|
|
198
|
-
}
|
|
199
|
-
const templateType = useChatTemplate ? state.modelConfig.chatTemplateType : null;
|
|
200
|
-
return formatChatMessages(messages, templateType, chatOptions);
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
function releasePerLayerInputBuffer(buffer, recorder, decodeBuffers, pleCache = null) {
|
|
204
|
-
if (!buffer) {
|
|
205
|
-
return;
|
|
206
|
-
}
|
|
207
|
-
const ownsBuffer = decodeBuffers?.ownsBuffer(buffer) ?? false;
|
|
208
|
-
if (ownsBuffer) {
|
|
209
|
-
return;
|
|
210
|
-
}
|
|
211
|
-
const cachedPleBuffer = pleCache?.ownedBuffers instanceof Set && pleCache.ownedBuffers.has(buffer);
|
|
212
|
-
if (cachedPleBuffer) {
|
|
213
|
-
return;
|
|
214
|
-
}
|
|
215
|
-
if (recorder) {
|
|
216
|
-
recorder.trackTemporaryBuffer(buffer);
|
|
217
|
-
return;
|
|
218
|
-
}
|
|
219
|
-
releaseBuffer(buffer);
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
function normalizePrefixEmbeddingOverride(override, hiddenSize, numTokens, contextLabel) {
|
|
223
|
-
if (override == null) {
|
|
224
|
-
return null;
|
|
225
|
-
}
|
|
226
|
-
if (typeof override !== 'object') {
|
|
227
|
-
throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides must be an object when provided.`);
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
const prefixLength = Number(override.prefixLength);
|
|
231
|
-
if (!Number.isFinite(prefixLength) || prefixLength < 0 || Math.floor(prefixLength) !== prefixLength) {
|
|
232
|
-
throw new Error(
|
|
233
|
-
`[Pipeline] ${contextLabel}: embeddingOverrides.prefixLength must be a non-negative integer.`
|
|
234
|
-
);
|
|
235
|
-
}
|
|
236
|
-
if (prefixLength === 0) {
|
|
237
|
-
return null;
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
const offset = Number(override.offset ?? 0);
|
|
241
|
-
if (!Number.isFinite(offset) || offset < 0 || Math.floor(offset) !== offset) {
|
|
242
|
-
throw new Error(
|
|
243
|
-
`[Pipeline] ${contextLabel}: embeddingOverrides.offset must be a non-negative integer.`
|
|
244
|
-
);
|
|
245
|
-
}
|
|
246
|
-
if (offset + prefixLength > numTokens) {
|
|
247
|
-
throw new Error(
|
|
248
|
-
`[Pipeline] ${contextLabel}: embedding override offset=${offset} + prefixLength=${prefixLength} exceeds numTokens=${numTokens}.`
|
|
249
|
-
);
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
const embeddings = override.embeddings ?? null;
|
|
253
|
-
if (!embeddings) {
|
|
254
|
-
throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides.embeddings is required when prefixLength > 0.`);
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
const expectedLength = prefixLength * hiddenSize;
|
|
258
|
-
if (embeddings instanceof Float32Array) {
|
|
259
|
-
if (embeddings.length !== expectedLength) {
|
|
260
|
-
throw new Error(
|
|
261
|
-
`[Pipeline] ${contextLabel}: embedding override length mismatch ` +
|
|
262
|
-
`(expected=${expectedLength}, got=${embeddings.length}).`
|
|
263
|
-
);
|
|
264
|
-
}
|
|
265
|
-
} else if (isGpuBufferInstance(embeddings)) {
|
|
266
|
-
const expectedBytes = expectedLength * Float32Array.BYTES_PER_ELEMENT;
|
|
267
|
-
if (embeddings.size < expectedBytes) {
|
|
268
|
-
throw new Error(
|
|
269
|
-
`[Pipeline] ${contextLabel}: embedding override GPUBuffer too small ` +
|
|
270
|
-
`(expected>=${expectedBytes} bytes, got=${embeddings.size}).`
|
|
271
|
-
);
|
|
272
|
-
}
|
|
273
|
-
} else {
|
|
274
|
-
throw new Error(
|
|
275
|
-
`[Pipeline] ${contextLabel}: embeddingOverrides.embeddings must be a Float32Array or GPUBuffer.`
|
|
276
|
-
);
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
return {
|
|
280
|
-
prefixLength,
|
|
281
|
-
offset,
|
|
282
|
-
embeddings,
|
|
283
|
-
expectedLength,
|
|
284
|
-
byteLength: expectedLength * Float32Array.BYTES_PER_ELEMENT,
|
|
285
|
-
byteOffset: offset * hiddenSize * Float32Array.BYTES_PER_ELEMENT,
|
|
286
|
-
};
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
function resolvePrefillEmbeddingInputIds(inputIds, embeddingInputSpan, contextLabel) {
|
|
290
|
-
if (embeddingInputSpan == null) {
|
|
291
|
-
return inputIds;
|
|
292
|
-
}
|
|
293
|
-
if (typeof embeddingInputSpan !== 'object') {
|
|
294
|
-
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan must be an object when provided.`);
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
const offset = Number(embeddingInputSpan.offset);
|
|
298
|
-
const length = Number(embeddingInputSpan.length);
|
|
299
|
-
const tokenId = Number(embeddingInputSpan.tokenId);
|
|
300
|
-
if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
|
|
301
|
-
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.offset must be a non-negative integer.`);
|
|
302
|
-
}
|
|
303
|
-
if (!Number.isFinite(length) || Math.floor(length) !== length || length < 0) {
|
|
304
|
-
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.length must be a non-negative integer.`);
|
|
305
|
-
}
|
|
306
|
-
if (!Number.isFinite(tokenId) || Math.floor(tokenId) !== tokenId || tokenId < 0) {
|
|
307
|
-
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.tokenId must be a non-negative integer.`);
|
|
308
|
-
}
|
|
309
|
-
if (offset + length > inputIds.length) {
|
|
310
|
-
throw new Error(
|
|
311
|
-
`[Pipeline] ${contextLabel}: embeddingInputSpan offset=${offset} + length=${length} ` +
|
|
312
|
-
`exceeds inputIds length ${inputIds.length}.`
|
|
313
|
-
);
|
|
314
|
-
}
|
|
315
|
-
const replacedInputIds = Array.from(inputIds);
|
|
316
|
-
replacedInputIds.fill(tokenId, offset, offset + length);
|
|
317
|
-
return replacedInputIds;
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
function resolvePrefillMultimodalBidirectionalSpan(inputIds, bidirectionalSpan, contextLabel) {
|
|
321
|
-
if (bidirectionalSpan == null) {
|
|
322
|
-
return null;
|
|
323
|
-
}
|
|
324
|
-
if (typeof bidirectionalSpan !== 'object') {
|
|
325
|
-
throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan must be an object when provided.`);
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
const offset = Number(bidirectionalSpan.offset);
|
|
329
|
-
const length = Number(bidirectionalSpan.length);
|
|
330
|
-
if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
|
|
331
|
-
throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.offset must be a non-negative integer.`);
|
|
332
|
-
}
|
|
333
|
-
if (!Number.isFinite(length) || Math.floor(length) !== length || length < 1) {
|
|
334
|
-
throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.length must be a positive integer.`);
|
|
335
|
-
}
|
|
336
|
-
if ((offset + length) > inputIds.length) {
|
|
337
|
-
throw new Error(
|
|
338
|
-
`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan offset=${offset} + length=${length} ` +
|
|
339
|
-
`exceeds inputIds length ${inputIds.length}.`
|
|
340
|
-
);
|
|
341
|
-
}
|
|
342
|
-
return { offset, length };
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenSize, contextLabel, executionPolicies = null) {
|
|
346
|
-
if (!override) {
|
|
347
|
-
return baseTensor;
|
|
348
|
-
}
|
|
349
|
-
if (baseTensor.dtype !== 'f32' && baseTensor.dtype !== 'f16') {
|
|
350
|
-
throw new Error(
|
|
351
|
-
`[Pipeline] ${contextLabel}: embedding overrides require f32 or f16 activations, got ${baseTensor.dtype}.`
|
|
352
|
-
);
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
const device = getDevice();
|
|
356
|
-
if (!device) {
|
|
357
|
-
throw new Error(`[Pipeline] ${contextLabel}: GPU device is required for embedding overrides.`);
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
const outputBuffer = acquireBuffer(baseTensor.buffer.size, undefined, 'prefill_embedding_override');
|
|
361
|
-
const targetElementOffset = override.offset * hiddenSize;
|
|
362
|
-
const targetByteOffset = targetElementOffset * (baseTensor.dtype === 'f16' ? 2 : 4);
|
|
363
|
-
try {
|
|
364
|
-
const encoder = device.createCommandEncoder();
|
|
365
|
-
encoder.copyBufferToBuffer(baseTensor.buffer, 0, outputBuffer, 0, baseTensor.buffer.size);
|
|
366
|
-
if (baseTensor.dtype === 'f32') {
|
|
367
|
-
if (isGpuBufferInstance(override.embeddings)) {
|
|
368
|
-
encoder.copyBufferToBuffer(override.embeddings, 0, outputBuffer, override.byteOffset, override.byteLength);
|
|
369
|
-
device.queue.submit([encoder.finish()]);
|
|
370
|
-
} else {
|
|
371
|
-
device.queue.submit([encoder.finish()]);
|
|
372
|
-
uploadData(outputBuffer, override.embeddings, override.byteOffset);
|
|
373
|
-
}
|
|
374
|
-
return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
|
|
375
|
-
}
|
|
376
|
-
|
|
377
|
-
if (isGpuBufferInstance(override.embeddings)) {
|
|
378
|
-
const overrideTensor = createTensor(
|
|
379
|
-
override.embeddings,
|
|
380
|
-
'f32',
|
|
381
|
-
[override.prefixLength, hiddenSize],
|
|
382
|
-
'prefill_embedding_override_f32'
|
|
383
|
-
);
|
|
384
|
-
assertImplicitDtypeTransitionAllowed({
|
|
385
|
-
executionPolicies,
|
|
386
|
-
fromDtype: 'f32',
|
|
387
|
-
toDtype: 'f16',
|
|
388
|
-
op: 'prefill_embedding_override',
|
|
389
|
-
detail: 'Prefix embedding override would pack GPU-provided f32 features into an f16 activation buffer.',
|
|
390
|
-
});
|
|
391
|
-
const castedOverride = await castF32ToF16(overrideTensor);
|
|
392
|
-
try {
|
|
393
|
-
encoder.copyBufferToBuffer(
|
|
394
|
-
castedOverride.buffer,
|
|
395
|
-
0,
|
|
396
|
-
outputBuffer,
|
|
397
|
-
targetByteOffset,
|
|
398
|
-
castedOverride.buffer.size
|
|
399
|
-
);
|
|
400
|
-
device.queue.submit([encoder.finish()]);
|
|
401
|
-
} finally {
|
|
402
|
-
releaseBuffer(castedOverride.buffer);
|
|
403
|
-
}
|
|
404
|
-
} else {
|
|
405
|
-
assertImplicitDtypeTransitionAllowed({
|
|
406
|
-
executionPolicies,
|
|
407
|
-
fromDtype: 'f32',
|
|
408
|
-
toDtype: 'f16',
|
|
409
|
-
op: 'prefill_embedding_override',
|
|
410
|
-
detail: 'Prefix embedding override would pack CPU-provided f32 features into an f16 activation buffer.',
|
|
411
|
-
});
|
|
412
|
-
const packedOverride = f32ToF16Array(override.embeddings);
|
|
413
|
-
device.queue.submit([encoder.finish()]);
|
|
414
|
-
uploadData(outputBuffer, packedOverride, targetByteOffset);
|
|
415
|
-
}
|
|
416
|
-
return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
|
|
417
|
-
} catch (error) {
|
|
418
|
-
releaseBuffer(outputBuffer);
|
|
419
|
-
throw error;
|
|
420
|
-
}
|
|
421
|
-
}
|
|
422
|
-
|
|
423
|
-
/**
|
|
424
|
-
* Resolve display text for a token ID sequence.
|
|
425
|
-
*
|
|
426
|
-
* Preference order:
|
|
427
|
-
* 1. renderTokenText (custom primary renderer) — if provided and returns non-empty string.
|
|
428
|
-
* 2. tokenizer.decode (default primary) — if renderTokenText is not provided.
|
|
429
|
-
* 3. renderFallbackTokenText (custom fallback) — if provided and returns non-empty string,
|
|
430
|
-
* unless primary returned empty string and fallback looks like a special token (<...>),
|
|
431
|
-
* in which case empty string is preserved to keep skip-special behavior deterministic.
|
|
432
|
-
* 4. tokenizer.decode with skipSpecialTokens=false (default fallback) — same special-token guard.
|
|
433
|
-
* 5. fallbackText — static fallback string (default '?').
|
|
434
|
-
*/
|
|
435
|
-
function resolveTokenText(tokenizer, tokenIds, fallbackText = '?', renderTokenText, renderFallbackTokenText) {
|
|
436
|
-
const renderPrimary = typeof renderTokenText === 'function'
|
|
437
|
-
? renderTokenText
|
|
438
|
-
: (ids) => tokenizer?.decode?.(ids);
|
|
439
|
-
const renderFallback = typeof renderFallbackTokenText === 'function'
|
|
440
|
-
? renderFallbackTokenText
|
|
441
|
-
: (ids) => tokenizer?.decode?.(ids, false);
|
|
442
|
-
|
|
443
|
-
const primaryText = renderPrimary(tokenIds);
|
|
444
|
-
if (typeof primaryText === 'string' && primaryText.length > 0) {
|
|
445
|
-
return primaryText;
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
const fallback = renderFallback(tokenIds);
|
|
449
|
-
if (typeof fallback === 'string' && fallback.length > 0) {
|
|
450
|
-
// Keep skip-special behavior deterministic: if primary decoding filtered this
|
|
451
|
-
// token to empty, do not reintroduce obvious special-token text via fallback.
|
|
452
|
-
if (
|
|
453
|
-
primaryText === ''
|
|
454
|
-
&& /^<[^>\n]{1,80}>$/.test(fallback.trim())
|
|
455
|
-
) {
|
|
456
|
-
return '';
|
|
457
|
-
}
|
|
458
|
-
return fallback;
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
return fallbackText;
|
|
462
|
-
}
|
|
463
|
-
|
|
464
|
-
function usesReplayPrefillDecode(state) {
|
|
465
|
-
return state?.modelConfig?.decodeStrategy === 'replay_prefill';
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
export function shouldDisablePrefillCommandBatching(state, opts, multimodalBidirectionalSpan) {
|
|
469
|
-
if (
|
|
470
|
-
opts?.disableCommandBatching === true
|
|
471
|
-
|| opts?.debug === true
|
|
472
|
-
|| (Array.isArray(opts?.debugLayers) && opts.debugLayers.length > 0)
|
|
473
|
-
) {
|
|
474
|
-
return true;
|
|
475
|
-
}
|
|
476
|
-
if (state?.kvCache?.layout === 'bdpa_paged') {
|
|
477
|
-
return true;
|
|
478
|
-
}
|
|
479
|
-
if (multimodalBidirectionalSpan == null) {
|
|
480
|
-
return false;
|
|
481
|
-
}
|
|
482
|
-
if (state?.kvCache?.hasGPUCache?.() !== true) {
|
|
483
|
-
return false;
|
|
484
|
-
}
|
|
485
|
-
// WORKAROUND: recorded prefill with live KV-cache writes regresses Gemma 4
|
|
486
|
-
// multimodal logits on the first generated token. Keep this lane fail-closed
|
|
487
|
-
// until the recorder/cache interaction is numerically matched to the direct path.
|
|
488
|
-
return true;
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
function assertIncrementalDecodeSupport(state, operation) {
|
|
492
|
-
if (!usesReplayPrefillDecode(state)) {
|
|
493
|
-
return;
|
|
494
|
-
}
|
|
495
|
-
throw new Error(
|
|
496
|
-
`[Pipeline] ${operation} is not supported for models that require replay-prefill decode. ` +
|
|
497
|
-
'Incremental KV-cache decode and prefix snapshots stay disabled when the model config does not resolve ' +
|
|
498
|
-
'explicit layerTypes for mixed-geometry/shared-KV decode.'
|
|
499
|
-
);
|
|
500
|
-
}
|
|
501
|
-
|
|
502
|
-
function summarizeExecutionPlan(plan) {
|
|
503
|
-
if (!plan) {
|
|
504
|
-
return null;
|
|
505
|
-
}
|
|
506
|
-
if (typeof plan !== 'object') {
|
|
507
|
-
log.warn('Pipeline', `summarizeExecutionPlan: expected object, got ${typeof plan}`);
|
|
508
|
-
return null;
|
|
509
|
-
}
|
|
510
|
-
if (typeof plan.id !== 'string') {
|
|
511
|
-
log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "id"');
|
|
512
|
-
}
|
|
513
|
-
if (typeof plan.activationDtype !== 'string') {
|
|
514
|
-
log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "activationDtype"');
|
|
515
|
-
}
|
|
516
|
-
return {
|
|
517
|
-
id: plan.id,
|
|
518
|
-
kernelPathId: plan.kernelPathId ?? null,
|
|
519
|
-
kernelPathSource: plan.kernelPathSource ?? 'none',
|
|
520
|
-
activationDtype: plan.activationDtype,
|
|
521
|
-
readbackInterval: plan.readbackInterval ?? null,
|
|
522
|
-
batchSize: plan.defaultBatchSize,
|
|
523
|
-
stopCheckMode: plan.defaultStopCheckMode,
|
|
524
|
-
ringTokens: plan.ringTokens ?? null,
|
|
525
|
-
ringStop: plan.ringStop ?? null,
|
|
526
|
-
ringStaging: plan.ringStaging ?? null,
|
|
527
|
-
};
|
|
528
|
-
}
|
|
529
|
-
|
|
530
|
-
export function shouldRetryWithFinitenessFallback(error) {
|
|
531
|
-
if (error?.name === 'FinitenessError') {
|
|
532
|
-
return true;
|
|
533
|
-
}
|
|
534
|
-
const message = typeof error?.message === 'string'
|
|
535
|
-
? error.message
|
|
536
|
-
: (typeof error === 'string' ? error : '');
|
|
537
|
-
if (!message.startsWith('[Sampling]')) {
|
|
538
|
-
return false;
|
|
539
|
-
}
|
|
540
|
-
return message.includes('no finite candidate logits after masking the pad token')
|
|
541
|
-
|| message.includes('Softmax produced no finite candidate probabilities');
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
function createUnhandledFinitenessPolicyError(state, contextLabel, error) {
|
|
545
|
-
const activePlan = resolveActiveExecutionPlan(state);
|
|
546
|
-
const wrapped = new Error(
|
|
547
|
-
`[Pipeline] ${contextLabel}: finiteness guard triggered for kernelPath ` +
|
|
548
|
-
`"${activePlan.kernelPathId ?? 'none'}" under fail-fast policy. ` +
|
|
549
|
-
'Resolve the unstable path with an explicit capability-aware execution override, ' +
|
|
550
|
-
'or opt into alternate-plan recovery with ' +
|
|
551
|
-
'runtime.inference.compute.rangeAwareSelectiveWidening.onTrigger="fallback-plan".',
|
|
552
|
-
error instanceof Error ? { cause: error } : undefined
|
|
553
|
-
);
|
|
554
|
-
wrapped.name = error?.name === 'FinitenessError' ? error.name : 'FinitenessError';
|
|
555
|
-
return wrapped;
|
|
556
|
-
}
|
|
557
|
-
|
|
558
|
-
function assertResolvedKVDtype(kvDtype, contextLabel) {
|
|
559
|
-
if (kvDtype === 'f16' || kvDtype === 'f32') {
|
|
560
|
-
return kvDtype;
|
|
561
|
-
}
|
|
562
|
-
throw new Error(
|
|
563
|
-
`[Pipeline] ${contextLabel}: expected execution-plan kvDtype to resolve to "f16" or "f32", ` +
|
|
564
|
-
`got ${JSON.stringify(kvDtype)}.`
|
|
565
|
-
);
|
|
566
|
-
}
|
|
567
|
-
|
|
568
|
-
function resolveTargetPlanKVDtype(plan, contextLabel) {
|
|
569
|
-
return assertResolvedKVDtype(
|
|
570
|
-
plan?.kernelPath?.kvDtype ?? plan?.activationDtype ?? null,
|
|
571
|
-
contextLabel
|
|
572
|
-
);
|
|
573
|
-
}
|
|
574
|
-
|
|
575
|
-
function resolveCurrentKVCacheDtype(state, plan, contextLabel) {
|
|
576
|
-
return assertResolvedKVDtype(
|
|
577
|
-
state?.kvCache?.kvDtype
|
|
578
|
-
?? plan?.kernelPath?.kvDtype
|
|
579
|
-
?? state?.runtimeConfig?.inference?.session?.kvcache?.kvDtype
|
|
580
|
-
?? plan?.activationDtype
|
|
581
|
-
?? null,
|
|
582
|
-
contextLabel
|
|
583
|
-
);
|
|
584
|
-
}
|
|
585
|
-
|
|
586
|
-
function cloneRuntimeInferenceWithKVDtype(state, kvDtype) {
|
|
587
|
-
const runtimeInference = state?.runtimeConfig?.inference;
|
|
588
|
-
if (!runtimeInference?.session?.kvcache) {
|
|
589
|
-
throw new Error(
|
|
590
|
-
'[Pipeline] runtime.inference.session.kvcache is required for finiteness fallback KV-cache recovery.'
|
|
591
|
-
);
|
|
592
|
-
}
|
|
593
|
-
return mergeRuntimeValues(runtimeInference, {
|
|
594
|
-
session: {
|
|
595
|
-
kvcache: {
|
|
596
|
-
kvDtype,
|
|
597
|
-
},
|
|
598
|
-
},
|
|
599
|
-
});
|
|
600
|
-
}
|
|
601
121
|
|
|
602
122
|
export class PipelineGenerator {
|
|
603
123
|
|
|
@@ -797,14 +317,36 @@ export class PipelineGenerator {
|
|
|
797
317
|
_sampleNextTokenFromLogits(logits, generatedIds, opts) {
|
|
798
318
|
const sampledLogits = Float32Array.from(logits);
|
|
799
319
|
applyRepetitionPenalty(sampledLogits, generatedIds, opts.repetitionPenalty);
|
|
320
|
+
// Optional pre-sample logit mask. Callers pass `opts.logitMaskFn` to
|
|
321
|
+
// implement grammar/schema-constrained decoding. The hook receives the
|
|
322
|
+
// mutable logit buffer (after repetition penalty) plus the running token
|
|
323
|
+
// sequence so it can track parse state across decode steps.
|
|
324
|
+
if (typeof opts?.logitMaskFn === "function") {
|
|
325
|
+
try {
|
|
326
|
+
opts.logitMaskFn(sampledLogits, {
|
|
327
|
+
generatedIds,
|
|
328
|
+
tokenizer: this.#state.tokenizer ?? null,
|
|
329
|
+
vocabSize: this.#state.modelConfig?.vocabSize ?? sampledLogits.length,
|
|
330
|
+
});
|
|
331
|
+
} catch (maskError) {
|
|
332
|
+
log.warn("Pipeline", `logitMaskFn threw; continuing without mask: ${maskError}`);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
800
335
|
const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
|
|
801
|
-
|
|
336
|
+
const tokenId = sample(sampledLogits, {
|
|
802
337
|
temperature: opts.temperature,
|
|
803
338
|
topP: opts.topP,
|
|
804
339
|
topK: opts.topK,
|
|
805
340
|
padTokenId,
|
|
806
341
|
seed: opts.seed,
|
|
807
342
|
});
|
|
343
|
+
if (typeof opts.onLogits === 'function') {
|
|
344
|
+
opts.onLogits(sampledLogits, {
|
|
345
|
+
tokenId,
|
|
346
|
+
inputTokenCount: Array.isArray(generatedIds) ? generatedIds.length : null,
|
|
347
|
+
});
|
|
348
|
+
}
|
|
349
|
+
return tokenId;
|
|
808
350
|
}
|
|
809
351
|
|
|
810
352
|
async _prefillPromptToLogits(prompt, opts, contextLabel) {
|
|
@@ -865,6 +407,11 @@ export class PipelineGenerator {
|
|
|
865
407
|
return this._matchesStopSequence(generatedIds, runtime.stopSequenceStart, opts.stopSequences);
|
|
866
408
|
}
|
|
867
409
|
|
|
410
|
+
_recordStopReason(reason, tokenId = null) {
|
|
411
|
+
this.#state.stats.stopReason = reason;
|
|
412
|
+
this.#state.stats.stopTokenId = Number.isInteger(tokenId) ? tokenId : null;
|
|
413
|
+
}
|
|
414
|
+
|
|
868
415
|
async *_generateTokensInternal(prompt, options = {}, mode = 'text') {
|
|
869
416
|
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
870
417
|
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
@@ -878,6 +425,8 @@ export class PipelineGenerator {
|
|
|
878
425
|
this.#state.stats.decodeRecordMs = 0;
|
|
879
426
|
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
880
427
|
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
428
|
+
this.#state.stats.prefillRecordMs = 0;
|
|
429
|
+
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
881
430
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
882
431
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
883
432
|
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
@@ -889,10 +438,14 @@ export class PipelineGenerator {
|
|
|
889
438
|
this.#state.stats.plePreparedTokenCacheBytes = 0;
|
|
890
439
|
this.#state.stats.decodeMode = null;
|
|
891
440
|
this.#state.stats.batchGuardReason = null;
|
|
441
|
+
this.#state.stats.stopReason = null;
|
|
442
|
+
this.#state.stats.stopTokenId = null;
|
|
892
443
|
this.#state.stats.ttftMs = 0;
|
|
893
444
|
const startTime = performance.now();
|
|
894
445
|
|
|
895
446
|
const opts = resolveGenerateOptions(this.#state, options);
|
|
447
|
+
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
448
|
+
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
896
449
|
// Validate and normalize sampling parameters through single source of truth
|
|
897
450
|
const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
|
|
898
451
|
opts.temperature = samplingConfig.temperature;
|
|
@@ -901,7 +454,8 @@ export class PipelineGenerator {
|
|
|
901
454
|
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
902
455
|
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
903
456
|
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
904
|
-
|
|
457
|
+
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
458
|
+
if (diagnosticsEnabled || tsirFixtureCfg) {
|
|
905
459
|
const captureConfig = {
|
|
906
460
|
...createDefaultCaptureConfig(),
|
|
907
461
|
enabled: true,
|
|
@@ -910,13 +464,20 @@ export class PipelineGenerator {
|
|
|
910
464
|
};
|
|
911
465
|
validateCaptureConfig(captureConfig);
|
|
912
466
|
this.#state.operatorDiagnostics = {
|
|
913
|
-
enabled: true,
|
|
467
|
+
enabled: diagnosticsEnabled === true,
|
|
914
468
|
captureConfig,
|
|
915
|
-
emitter: new OperatorEventEmitter({
|
|
469
|
+
emitter: diagnosticsEnabled ? new OperatorEventEmitter({
|
|
916
470
|
modelHash: this.#state.manifest?.modelId ?? null,
|
|
917
471
|
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
918
472
|
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
919
|
-
}),
|
|
473
|
+
}) : null,
|
|
474
|
+
tsirFixture: tsirFixtureCfg ? {
|
|
475
|
+
dir: tsirFixtureCfg.dir,
|
|
476
|
+
layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
|
|
477
|
+
? tsirFixtureCfg.layerFilter
|
|
478
|
+
: null,
|
|
479
|
+
records: [],
|
|
480
|
+
} : null,
|
|
920
481
|
};
|
|
921
482
|
}
|
|
922
483
|
const activePlan = opts.executionPlan ?? resolveActiveExecutionPlan(this.#state);
|
|
@@ -1036,6 +597,7 @@ export class PipelineGenerator {
|
|
|
1036
597
|
yield* emitToken(this, firstToken, decodeToken);
|
|
1037
598
|
|
|
1038
599
|
if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
|
|
600
|
+
this._recordStopReason('stop-token-or-sequence', firstToken);
|
|
1039
601
|
this.#state.stats.decodeTimeMs = 0;
|
|
1040
602
|
this.#state.stats.tokensGenerated = 1;
|
|
1041
603
|
this.#state.stats.decodeTokens = 1;
|
|
@@ -1262,6 +824,31 @@ export class PipelineGenerator {
|
|
|
1262
824
|
// Generation Public API
|
|
1263
825
|
// ==========================================================================
|
|
1264
826
|
|
|
827
|
+
/**
|
|
828
|
+
* Truncate the KV cache back to `seqLen` tokens and set `currentSeqLen` to
|
|
829
|
+
* match. Intended for "prefix-reuse" workflows where a caller wants to run
|
|
830
|
+
* several decodes that share a common prompt prefix: prefill once with the
|
|
831
|
+
* shared prefix, decode the first tail, then `resetToSeqLen(prefixLen)` to
|
|
832
|
+
* drop the tail's KV entries and reuse the prefix KV for the next tail.
|
|
833
|
+
*
|
|
834
|
+
* Only valid when no decode is in progress.
|
|
835
|
+
*/
|
|
836
|
+
resetToSeqLen(seqLen) {
|
|
837
|
+
if (this.#state.isGenerating) {
|
|
838
|
+
throw new Error('InferencePipeline.resetToSeqLen: cannot reset while generation is in progress');
|
|
839
|
+
}
|
|
840
|
+
const target = Math.max(0, Math.floor(Number(seqLen) || 0));
|
|
841
|
+
if (!Number.isFinite(target)) {
|
|
842
|
+
throw new Error('InferencePipeline.resetToSeqLen: seqLen must be a finite non-negative integer');
|
|
843
|
+
}
|
|
844
|
+
if (target > this.#state.currentSeqLen) {
|
|
845
|
+
throw new Error(
|
|
846
|
+
`InferencePipeline.resetToSeqLen: target ${target} exceeds currentSeqLen ${this.#state.currentSeqLen}`
|
|
847
|
+
);
|
|
848
|
+
}
|
|
849
|
+
this.#state.kvCache?.truncate?.(target);
|
|
850
|
+
this.#state.currentSeqLen = target;
|
|
851
|
+
}
|
|
1265
852
|
|
|
1266
853
|
async *generate(prompt, options = {}) {
|
|
1267
854
|
yield* this._generateTokensInternal(prompt, options, 'text');
|
|
@@ -1284,6 +871,8 @@ export class PipelineGenerator {
|
|
|
1284
871
|
this.#state.stats.decodeRecordMs = 0;
|
|
1285
872
|
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
1286
873
|
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
874
|
+
this.#state.stats.prefillRecordMs = 0;
|
|
875
|
+
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
1287
876
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
1288
877
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
1289
878
|
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
@@ -1295,15 +884,46 @@ export class PipelineGenerator {
|
|
|
1295
884
|
this.#state.stats.plePreparedTokenCacheBytes = 0;
|
|
1296
885
|
this.#state.stats.decodeMode = null;
|
|
1297
886
|
this.#state.stats.batchGuardReason = null;
|
|
887
|
+
this.#state.stats.stopReason = null;
|
|
888
|
+
this.#state.stats.stopTokenId = null;
|
|
1298
889
|
this.#state.stats.ttftMs = 0;
|
|
1299
890
|
const startTime = performance.now();
|
|
1300
891
|
const opts = resolveGenerateOptions(this.#state, options);
|
|
892
|
+
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
1301
893
|
// Validate and normalize sampling parameters through single source of truth
|
|
1302
894
|
const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
|
|
1303
895
|
opts.temperature = samplingConfig.temperature;
|
|
1304
896
|
opts.topP = samplingConfig.topP;
|
|
1305
897
|
opts.topK = samplingConfig.topK;
|
|
1306
898
|
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
899
|
+
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
900
|
+
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
901
|
+
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
902
|
+
if (diagnosticsEnabled || tsirFixtureCfg) {
|
|
903
|
+
const captureConfig = {
|
|
904
|
+
...createDefaultCaptureConfig(),
|
|
905
|
+
enabled: true,
|
|
906
|
+
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
907
|
+
...(options?.diagnostics?.captureConfig ?? {}),
|
|
908
|
+
};
|
|
909
|
+
validateCaptureConfig(captureConfig);
|
|
910
|
+
this.#state.operatorDiagnostics = {
|
|
911
|
+
enabled: diagnosticsEnabled === true,
|
|
912
|
+
captureConfig,
|
|
913
|
+
emitter: diagnosticsEnabled ? new OperatorEventEmitter({
|
|
914
|
+
modelHash: this.#state.manifest?.modelId ?? null,
|
|
915
|
+
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
916
|
+
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
917
|
+
}) : null,
|
|
918
|
+
tsirFixture: tsirFixtureCfg ? {
|
|
919
|
+
dir: tsirFixtureCfg.dir,
|
|
920
|
+
layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
|
|
921
|
+
? tsirFixtureCfg.layerFilter
|
|
922
|
+
: null,
|
|
923
|
+
records: [],
|
|
924
|
+
} : null,
|
|
925
|
+
};
|
|
926
|
+
}
|
|
1307
927
|
|
|
1308
928
|
try {
|
|
1309
929
|
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
@@ -1353,6 +973,7 @@ export class PipelineGenerator {
|
|
|
1353
973
|
tokenIds.push(tokenId);
|
|
1354
974
|
}
|
|
1355
975
|
} else {
|
|
976
|
+
this._recordStopReason('stop-token-or-sequence', firstToken);
|
|
1356
977
|
this.#state.stats.decodeTimeMs = 0;
|
|
1357
978
|
this.#state.stats.tokensGenerated = 1;
|
|
1358
979
|
this.#state.stats.decodeTokens = 1;
|
|
@@ -1370,6 +991,14 @@ export class PipelineGenerator {
|
|
|
1370
991
|
this.#state.stats.executionPlan.finalActivePlanId = this.#state.executionPlanState?.activePlanId ?? null;
|
|
1371
992
|
}
|
|
1372
993
|
resetActiveExecutionPlan(this.#state);
|
|
994
|
+
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
995
|
+
? {
|
|
996
|
+
enabled: true,
|
|
997
|
+
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
998
|
+
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
999
|
+
}
|
|
1000
|
+
: null;
|
|
1001
|
+
this.#state.operatorDiagnostics = null;
|
|
1373
1002
|
this.#state.isGenerating = false;
|
|
1374
1003
|
}
|
|
1375
1004
|
}
|
|
@@ -1683,6 +1312,7 @@ export class PipelineGenerator {
|
|
|
1683
1312
|
if (options.onToken) options.onToken(firstToken, firstText);
|
|
1684
1313
|
|
|
1685
1314
|
if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
|
|
1315
|
+
this._recordStopReason('stop-token-or-sequence', firstToken);
|
|
1686
1316
|
this.#state.stats.decodeTimeMs = 0;
|
|
1687
1317
|
this.#state.stats.tokensGenerated = 1;
|
|
1688
1318
|
this.#state.stats.decodeTokens = 1;
|
|
@@ -1729,6 +1359,10 @@ export class PipelineGenerator {
|
|
|
1729
1359
|
}
|
|
1730
1360
|
|
|
1731
1361
|
const decodeStart = performance.now();
|
|
1362
|
+
const resolvedPerLayerInputsSession = resolvePerLayerInputsSession(
|
|
1363
|
+
this.#state.modelConfig.perLayerInputsSession ?? null,
|
|
1364
|
+
this.#state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
1365
|
+
);
|
|
1732
1366
|
const lmHead = this.#state.weights.get('lm_head');
|
|
1733
1367
|
const embedBuffer = this.#state.weights.get('embed');
|
|
1734
1368
|
const hasCpuWeights = isCpuWeightBuffer(lmHead)
|
|
@@ -1805,7 +1439,10 @@ export class PipelineGenerator {
|
|
|
1805
1439
|
}
|
|
1806
1440
|
|
|
1807
1441
|
while (tokensGenerated < opts.maxTokens) {
|
|
1808
|
-
if (options.signal?.aborted)
|
|
1442
|
+
if (options.signal?.aborted) {
|
|
1443
|
+
this._recordStopReason('aborted');
|
|
1444
|
+
break;
|
|
1445
|
+
}
|
|
1809
1446
|
if (this._hasFinitenessFallbackWindow() && useBatchPath) {
|
|
1810
1447
|
useBatchPath = false;
|
|
1811
1448
|
}
|
|
@@ -1854,9 +1491,15 @@ export class PipelineGenerator {
|
|
|
1854
1491
|
if (batchTokens.length > 0 && options.onBatch) options.onBatch(batchTokens);
|
|
1855
1492
|
if (opts.stopSequences.length > 0) {
|
|
1856
1493
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
1857
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq)))
|
|
1494
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
1495
|
+
this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
|
|
1496
|
+
break;
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
if (hitStop) {
|
|
1500
|
+
this._recordStopReason('stop-token');
|
|
1501
|
+
break;
|
|
1858
1502
|
}
|
|
1859
|
-
if (hitStop) break;
|
|
1860
1503
|
if (shouldDisableBatchDecodeAfterShortBatch({
|
|
1861
1504
|
hitStop,
|
|
1862
1505
|
actualCount: batchResult.actualCount,
|
|
@@ -1894,7 +1537,10 @@ export class PipelineGenerator {
|
|
|
1894
1537
|
if (options.onToken) options.onToken(nextToken, tokenText);
|
|
1895
1538
|
}
|
|
1896
1539
|
this._consumeFinitenessFallbackToken(opts);
|
|
1897
|
-
if (isStopToken(nextToken, stopTokenIds, eosToken))
|
|
1540
|
+
if (isStopToken(nextToken, stopTokenIds, eosToken)) {
|
|
1541
|
+
this._recordStopReason('stop-token', nextToken);
|
|
1542
|
+
break;
|
|
1543
|
+
}
|
|
1898
1544
|
}
|
|
1899
1545
|
} else if (opts.speculation?.mode === 'self') {
|
|
1900
1546
|
// Self-speculation: decode one base token plus a configurable burst of
|
|
@@ -1932,11 +1578,17 @@ export class PipelineGenerator {
|
|
|
1932
1578
|
}
|
|
1933
1579
|
this._consumeFinitenessFallbackToken(opts);
|
|
1934
1580
|
|
|
1935
|
-
if (isStopToken(baseToken, stopTokenIds, eosToken))
|
|
1581
|
+
if (isStopToken(baseToken, stopTokenIds, eosToken)) {
|
|
1582
|
+
this._recordStopReason('stop-token', baseToken);
|
|
1583
|
+
break;
|
|
1584
|
+
}
|
|
1936
1585
|
if (tokensGenerated >= opts.maxTokens) break;
|
|
1937
1586
|
if (opts.stopSequences.length > 0) {
|
|
1938
1587
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
1939
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq)))
|
|
1588
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
1589
|
+
this._recordStopReason('stop-sequence', baseToken);
|
|
1590
|
+
break;
|
|
1591
|
+
}
|
|
1940
1592
|
}
|
|
1941
1593
|
|
|
1942
1594
|
for (let specIndex = 0; specIndex < speculativeBurstTokens; specIndex += 1) {
|
|
@@ -1975,19 +1627,27 @@ export class PipelineGenerator {
|
|
|
1975
1627
|
}
|
|
1976
1628
|
|
|
1977
1629
|
if (isStopToken(specToken, stopTokenIds, eosToken)) {
|
|
1630
|
+
this._recordStopReason('stop-token', specToken);
|
|
1978
1631
|
break;
|
|
1979
1632
|
}
|
|
1980
1633
|
if (opts.stopSequences.length > 0) {
|
|
1981
1634
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
1982
1635
|
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
1636
|
+
this._recordStopReason('stop-sequence', specToken);
|
|
1983
1637
|
break;
|
|
1984
1638
|
}
|
|
1985
1639
|
}
|
|
1986
1640
|
}
|
|
1987
|
-
if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken))
|
|
1641
|
+
if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken)) {
|
|
1642
|
+
this._recordStopReason('stop-token', generatedIds[generatedIds.length - 1]);
|
|
1643
|
+
break;
|
|
1644
|
+
}
|
|
1988
1645
|
if (opts.stopSequences.length > 0) {
|
|
1989
1646
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
1990
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq)))
|
|
1647
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
1648
|
+
this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
|
|
1649
|
+
break;
|
|
1650
|
+
}
|
|
1991
1651
|
}
|
|
1992
1652
|
} else {
|
|
1993
1653
|
const tokenStart = performance.now();
|
|
@@ -2020,7 +1680,7 @@ export class PipelineGenerator {
|
|
|
2020
1680
|
nextToken,
|
|
2021
1681
|
pleWeights.embedTokensPerLayer,
|
|
2022
1682
|
this.#state.modelConfig.numLayers * pleHiddenSize,
|
|
2023
|
-
|
|
1683
|
+
resolvedPerLayerInputsSession,
|
|
2024
1684
|
);
|
|
2025
1685
|
}
|
|
2026
1686
|
}
|
|
@@ -2041,14 +1701,23 @@ export class PipelineGenerator {
|
|
|
2041
1701
|
log.debug('Decode', `#${tokensGenerated} "${tokenText}" ${tokenTime.toFixed(0)}ms (${tokPerSec.toFixed(2)} tok/s avg)`);
|
|
2042
1702
|
}
|
|
2043
1703
|
|
|
2044
|
-
if (isStopToken(nextToken, stopTokenIds, eosToken))
|
|
1704
|
+
if (isStopToken(nextToken, stopTokenIds, eosToken)) {
|
|
1705
|
+
this._recordStopReason('stop-token', nextToken);
|
|
1706
|
+
break;
|
|
1707
|
+
}
|
|
2045
1708
|
if (opts.stopSequences.length > 0) {
|
|
2046
1709
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
2047
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq)))
|
|
1710
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
1711
|
+
this._recordStopReason('stop-sequence', nextToken);
|
|
1712
|
+
break;
|
|
1713
|
+
}
|
|
2048
1714
|
}
|
|
2049
1715
|
}
|
|
2050
1716
|
}
|
|
2051
1717
|
|
|
1718
|
+
if (!this.#state.stats.stopReason) {
|
|
1719
|
+
this._recordStopReason(tokensGenerated >= opts.maxTokens ? 'max-tokens' : 'completed');
|
|
1720
|
+
}
|
|
2052
1721
|
this.#state.stats.decodeTimeMs = performance.now() - decodeStart;
|
|
2053
1722
|
this.#state.stats.tokensGenerated = tokensGenerated;
|
|
2054
1723
|
this.#state.stats.decodeTokens = tokensGenerated;
|
|
@@ -2214,6 +1883,7 @@ export class PipelineGenerator {
|
|
|
2214
1883
|
context,
|
|
2215
1884
|
{
|
|
2216
1885
|
numTokens,
|
|
1886
|
+
pleCache: this.#state.pleCache ?? null,
|
|
2217
1887
|
}
|
|
2218
1888
|
);
|
|
2219
1889
|
} catch (error) {
|
|
@@ -2273,16 +1943,35 @@ export class PipelineGenerator {
|
|
|
2273
1943
|
// Chunked recorder submission: submit every N layers to release tracked intermediate
|
|
2274
1944
|
// buffers, preventing unbounded memory growth during large prefills. Critical for
|
|
2275
1945
|
// replay_prefill models where each decode step re-runs a prefill-style layer pass.
|
|
2276
|
-
const
|
|
1946
|
+
const prefillRecorderChunkLayers = resolvePrefillRecorderChunkLayers({
|
|
1947
|
+
hasGpuSplitPerLayerInputs: context.perLayerInputsSession?.materialization === 'gpu_split_tables',
|
|
1948
|
+
numTokens,
|
|
1949
|
+
});
|
|
2277
1950
|
|
|
2278
1951
|
let currentHiddenBuffer = hiddenStates.buffer;
|
|
1952
|
+
let prefillRecordMs = 0;
|
|
1953
|
+
let prefillSubmitWaitMs = 0;
|
|
2279
1954
|
try {
|
|
2280
1955
|
for (let l = 0; l < config.numLayers; l++) {
|
|
1956
|
+
// Per-layer hard cancellation: when the caller's AbortSignal aborts,
|
|
1957
|
+
// exit the prefill loop between layer dispatches rather than
|
|
1958
|
+
// continuing to burn GPU on superseded work. Granularity is one
|
|
1959
|
+
// layer (~5-50ms), which is the fastest cancel granularity WebGPU
|
|
1960
|
+
// exposes today.
|
|
1961
|
+
if (opts?.signal?.aborted) {
|
|
1962
|
+
const reason = typeof opts.signal.reason === "string" ? opts.signal.reason : "Doppler: prefill aborted";
|
|
1963
|
+
const err = new Error(reason);
|
|
1964
|
+
err.name = "AbortError";
|
|
1965
|
+
err.code = "ABORT_ERR";
|
|
1966
|
+
throw err;
|
|
1967
|
+
}
|
|
2281
1968
|
context.recorder = currentRecorder;
|
|
2282
1969
|
context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
|
|
2283
1970
|
|
|
2284
1971
|
const prevBuffer = currentHiddenBuffer;
|
|
1972
|
+
const layerRecordStart = performance.now();
|
|
2285
1973
|
const layerOutput = await processLayer(l, currentHiddenBuffer, numTokens, true, context);
|
|
1974
|
+
prefillRecordMs += performance.now() - layerRecordStart;
|
|
2286
1975
|
if (!isGpuBufferInstance(layerOutput)) throw new Error('Expected GPUBuffer from processLayer');
|
|
2287
1976
|
currentHiddenBuffer = layerOutput;
|
|
2288
1977
|
releasePerLayerInputBuffer(
|
|
@@ -2300,7 +1989,7 @@ export class PipelineGenerator {
|
|
|
2300
1989
|
const isChunkBoundary = !isCheckpoint
|
|
2301
1990
|
&& currentRecorder
|
|
2302
1991
|
&& l < config.numLayers - 1
|
|
2303
|
-
&& (l + 1) %
|
|
1992
|
+
&& (l + 1) % prefillRecorderChunkLayers === 0;
|
|
2304
1993
|
|
|
2305
1994
|
if (isCheckpoint && currentRecorder) {
|
|
2306
1995
|
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
@@ -2360,8 +2049,25 @@ export class PipelineGenerator {
|
|
|
2360
2049
|
currentRecorder,
|
|
2361
2050
|
'prefill_chunk_carry'
|
|
2362
2051
|
);
|
|
2363
|
-
|
|
2364
|
-
|
|
2052
|
+
// Chunk boundary exists only to bound intermediate buffer lifetime.
|
|
2053
|
+
// When the runtime opts into async chunk submit AND profile timings
|
|
2054
|
+
// are not being collected, skip the CPU-GPU wait: queue order is
|
|
2055
|
+
// preserved across submits and deferred cleanup still releases
|
|
2056
|
+
// tracked buffers when GPU work completes. Profile runs keep the
|
|
2057
|
+
// sync path because resolveProfileTimings requires the submitted
|
|
2058
|
+
// work to be done.
|
|
2059
|
+
const manifestSubmit = this.#state.modelConfig?.sessionSettings?.prefillChunkSubmitMode;
|
|
2060
|
+
const chunkSubmitMode = (manifestSubmit !== null && manifestSubmit !== undefined)
|
|
2061
|
+
? manifestSubmit
|
|
2062
|
+
: (this.#state.runtimeConfig?.inference?.session?.prefillChunkSubmitMode ?? 'sync');
|
|
2063
|
+
const chunkSubmitStart = performance.now();
|
|
2064
|
+
if (chunkSubmitMode === 'async' && !opts.profile) {
|
|
2065
|
+
currentRecorder.submit({ cleanup: 'deferred' });
|
|
2066
|
+
} else {
|
|
2067
|
+
await currentRecorder.submitAndWait();
|
|
2068
|
+
await recordProfile(currentRecorder);
|
|
2069
|
+
}
|
|
2070
|
+
prefillSubmitWaitMs += performance.now() - chunkSubmitStart;
|
|
2365
2071
|
currentRecorder = createRecorder('prefill-chunk');
|
|
2366
2072
|
}
|
|
2367
2073
|
}
|
|
@@ -2378,8 +2084,26 @@ export class PipelineGenerator {
|
|
|
2378
2084
|
}
|
|
2379
2085
|
}
|
|
2380
2086
|
releaseSharedAttentionState(context.sharedAttentionState, currentRecorder);
|
|
2087
|
+
this.#state.stats.prefillRecordMs = (this.#state.stats.prefillRecordMs ?? 0) + prefillRecordMs;
|
|
2088
|
+
this.#state.stats.prefillSubmitWaitMs = (this.#state.stats.prefillSubmitWaitMs ?? 0) + prefillSubmitWaitMs;
|
|
2381
2089
|
}
|
|
2382
2090
|
|
|
2091
|
+
{
|
|
2092
|
+
const tsirFixtureCfg = this.#state.operatorDiagnostics?.tsirFixture ?? null;
|
|
2093
|
+
if (tsirFixtureCfg && Array.isArray(tsirFixtureCfg.pendingReads) && tsirFixtureCfg.pendingReads.length > 0) {
|
|
2094
|
+
if (currentRecorder) {
|
|
2095
|
+
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
2096
|
+
currentHiddenBuffer,
|
|
2097
|
+
currentRecorder,
|
|
2098
|
+
'prefill_tsir_drain_carry'
|
|
2099
|
+
);
|
|
2100
|
+
await currentRecorder.submitAndWait();
|
|
2101
|
+
await recordProfile(currentRecorder);
|
|
2102
|
+
currentRecorder = undefined;
|
|
2103
|
+
}
|
|
2104
|
+
await drainPendingTsirReads(tsirFixtureCfg);
|
|
2105
|
+
}
|
|
2106
|
+
}
|
|
2383
2107
|
if (this.#state.finitenessBuffer) {
|
|
2384
2108
|
if (currentRecorder) {
|
|
2385
2109
|
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|