doppler-gpu 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +210 -0
- package/README.md +63 -38
- package/assets/doppler.svg +57 -1601
- package/package.json +111 -32
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +486 -0
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +885 -744
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +469 -43
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +66 -10
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +69 -4
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +34 -50
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +35 -2
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +148 -3
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +8 -60
- package/src/client/wrap-pipeline-handle.d.ts +82 -0
- package/src/client/wrap-pipeline-handle.js +259 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +28 -157
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +163 -126
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +3593 -1146
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +3 -1
- package/src/config/merge-helpers.js +2 -1
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +65 -7
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +20 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +16 -0
- package/src/config/schema/converter.schema.js +11 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +13 -2
- package/src/config/schema/doppler.schema.js +140 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +39 -1
- package/src/config/schema/execution-v1.schema.js +35 -20
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +14 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +175 -2
- package/src/config/schema/manifest.schema.js +58 -3
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -9
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +183 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
- package/src/config/transforms/execution-graph-transforms.js +829 -168
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +31 -14
- package/src/converter/core.d.ts +65 -0
- package/src/converter/core.js +1153 -22
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +218 -5
- package/src/converter/quantizer.d.ts +24 -4
- package/src/converter/quantizer.js +92 -2
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +296 -38
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1733 -58
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +5 -14
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +192 -12
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +111 -2
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +10 -0
- package/src/formats/rdrr/index.js +10 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +26 -6
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +193 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +111 -2
- package/src/formats/rdrr/validation.js +260 -1
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +45 -23
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +130 -14
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +1 -8
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +458 -16
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +13 -120
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +84 -8
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +25 -22
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +77 -5
- package/src/gpu/kernels/index.js +55 -4
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +44 -2
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +172 -10
- package/src/gpu/kernels/matmul.d.ts +1 -29
- package/src/gpu/kernels/matmul.js +240 -38
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +1 -56
- package/src/gpu/kernels/moe.js +46 -33
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -12
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +6 -4
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +30 -7
- package/src/gpu/kernels/rmsnorm.js +49 -8
- package/src/gpu/kernels/rmsnorm.wgsl +15 -12
- package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +28 -21
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +8 -223
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +43 -0
- package/src/gpu/kernels/utils.js +81 -11
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +10 -12
- package/src/gpu/uniform-cache.js +25 -30
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
- package/src/inference/browser-harness-suite-helpers.js +525 -3
- package/src/inference/browser-harness-text-helpers.d.ts +89 -0
- package/src/inference/browser-harness-text-helpers.js +459 -25
- package/src/inference/browser-harness.d.ts +7 -28
- package/src/inference/browser-harness.js +556 -79
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +31 -0
- package/src/inference/kv-cache/base.js +125 -2
- package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
- package/src/inference/kv-cache/mixed-geometry.js +121 -4
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
- package/src/inference/pipelines/text/attention/projections.js +398 -71
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +542 -56
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +343 -50
- package/src/inference/pipelines/text/attention/types.d.ts +7 -1
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +25 -26
- package/src/inference/pipelines/text/config.js +304 -25
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +25 -29
- package/src/inference/pipelines/text/embed.js +16 -45
- package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
- package/src/inference/pipelines/text/execution-plan.js +21 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
- package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
- package/src/inference/pipelines/text/execution-v1.js +461 -158
- package/src/inference/pipelines/text/ffn/dense.js +214 -6
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +84 -17
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +9 -7
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
- package/src/inference/pipelines/text/generator-steps.js +275 -79
- package/src/inference/pipelines/text/generator.d.ts +38 -5
- package/src/inference/pipelines/text/generator.js +1259 -496
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +13 -3
- package/src/inference/pipelines/text/init.js +213 -725
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +373 -558
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +419 -66
- package/src/inference/pipelines/text/logits/index.d.ts +3 -3
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +42 -4
- package/src/inference/pipelines/text/ops.js +20 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +119 -31
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +153 -9
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
- package/src/inference/pipelines/text/types.d.ts +92 -81
- package/src/inference/pipelines/text/weights.d.ts +4 -52
- package/src/inference/pipelines/text/weights.js +6 -56
- package/src/inference/pipelines/text.d.ts +33 -1
- package/src/inference/pipelines/text.js +375 -55
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +341 -174
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +14 -0
- package/src/loader/doppler-loader.js +378 -25
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +276 -11
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +61 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +52 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +36 -8
- package/src/loader/tensors/tensor-loader.js +521 -22
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +13 -42
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +57 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +162 -10
- package/src/rules/inference/capability-transforms.rules.json +111 -0
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +117 -16
- package/src/rules/inference/ffn.rules.json +24 -2
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +17 -0
- package/src/rules/rule-registry.js +59 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +241 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +31 -4
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +53 -9
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +8 -3
- package/src/tooling/command-api.d.ts +22 -1
- package/src/tooling/command-api.js +11 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +14 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +3 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +571 -37
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +107 -17
- package/src/tooling/node-command-runner.js +43 -2
- package/src/tooling/node-convert-worker-pool.js +15 -0
- package/src/tooling/node-convert-worker.js +11 -1
- package/src/tooling/node-converter.js +102 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +112 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +10 -5
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +110 -8
- package/src/tooling/source-runtime-bundle.d.ts +3 -0
- package/src/tooling/source-runtime-bundle.js +30 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +25 -12
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -7,27 +7,34 @@ import { markWarmed as markKernelCacheWarmed } from '../../../gpu/kernel-selecti
|
|
|
7
7
|
import { resetSubmitStats, logSubmitStats } from '../../../gpu/submit-tracker.js';
|
|
8
8
|
import { createCommandRecorder, createProfilingRecorder, CommandRecorder } from '../../../gpu/command-recorder.js';
|
|
9
9
|
import { allowReadback } from '../../../gpu/perf-guards.js';
|
|
10
|
-
import { log, trace } from '../../../debug/index.js';
|
|
10
|
+
import { log, trace, isTraceEnabled } from '../../../debug/index.js';
|
|
11
|
+
import {
|
|
12
|
+
runMatmul,
|
|
13
|
+
runRMSNorm,
|
|
14
|
+
runGeLU,
|
|
15
|
+
runResidualAdd,
|
|
16
|
+
runScale,
|
|
17
|
+
runSoftmax,
|
|
18
|
+
runSoftEmbeddingSplitF16,
|
|
19
|
+
runSoftEmbeddingLogitsF16,
|
|
20
|
+
} from '../../../gpu/kernel-selector.js';
|
|
21
|
+
import { runDiffusionGemmaCanvasStats } from '../../../gpu/kernels/diffusion-gemma-sampling.js';
|
|
11
22
|
import {
|
|
12
23
|
CAPTURE_LEVELS,
|
|
13
24
|
createDefaultCaptureConfig,
|
|
14
25
|
validateCaptureConfig,
|
|
15
26
|
} from '../../../debug/index.js';
|
|
16
27
|
import { validateCallTimeOptions } from '../../../config/param-validator.js';
|
|
17
|
-
import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
|
|
18
28
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
19
|
-
import { castF32ToF16 } from '../../../gpu/kernels/cast.js';
|
|
20
|
-
import { f32ToF16Array } from '../../kv-cache/types.js';
|
|
21
29
|
|
|
22
30
|
// Pipeline sub-modules
|
|
23
31
|
import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './sampling.js';
|
|
24
|
-
import {
|
|
25
|
-
import { formatChatMessages } from './chat-format.js';
|
|
32
|
+
import { createKVCache, isStopToken } from './init.js';
|
|
26
33
|
import { embed } from './embed.js';
|
|
27
34
|
import { processLayer } from './layer.js';
|
|
28
|
-
import { computeLogits, recordLogitsGPU, extractLastPositionLogits, applySoftcapping } from './logits/index.js';
|
|
35
|
+
import { computeLogits, computeLogitsGPU, recordLogitsGPU, extractLastPositionLogits, applySoftcapping } from './logits/index.js';
|
|
29
36
|
import { OperatorEventEmitter } from './operator-events.js';
|
|
30
|
-
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, getWeightDtype } from '../../../gpu/weight-buffer.js';
|
|
37
|
+
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype, getWeightMetadata, getLayout } from '../../../gpu/weight-buffer.js';
|
|
31
38
|
import {
|
|
32
39
|
decodeStep,
|
|
33
40
|
decodeStepLogits,
|
|
@@ -62,12 +69,14 @@ import { resolveSamplingConfig } from './sampling-config.js';
|
|
|
62
69
|
import { decodeReadback, getLogitsHealth } from './debug-utils/index.js';
|
|
63
70
|
import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
|
|
64
71
|
import { resolveDeferredRoundingWindowTokens } from './finiteness-policy.js';
|
|
72
|
+
import { drainPendingTsirReads } from './tsir-fixture-writer.js';
|
|
65
73
|
import {
|
|
66
74
|
activateFallbackExecutionPlan,
|
|
67
75
|
hasFallbackExecutionPlan,
|
|
68
76
|
rebaseExecutionSessionPlan,
|
|
69
77
|
resetActiveExecutionPlan,
|
|
70
78
|
resolveMaxBatchDecodeTokens,
|
|
79
|
+
resolvePrefillRecorderChunkLayers,
|
|
71
80
|
resolveActiveExecutionPlan,
|
|
72
81
|
setActiveExecutionPlan,
|
|
73
82
|
} from './execution-plan.js';
|
|
@@ -81,538 +90,306 @@ import {
|
|
|
81
90
|
preparePerLayerInputs,
|
|
82
91
|
createPleBufferCache,
|
|
83
92
|
prefetchPerLayerRow,
|
|
84
|
-
ensurePleScaledProjectionNormWeight,
|
|
85
|
-
ensurePleGpuHotVocabularyRuntime,
|
|
86
|
-
ensurePleGpuSplitTablesRuntime,
|
|
87
93
|
getPleHotVocabularyRuntime,
|
|
88
94
|
hasGpuSplitPerLayerInputEmbeddings,
|
|
89
95
|
hasRangeBackedPerLayerInputEmbeddings,
|
|
90
96
|
} from './per-layer-inputs.js';
|
|
91
97
|
import { createTensor } from '../../../gpu/tensor.js';
|
|
92
|
-
import {
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
intentBundleModulePromise ??= import('../../../experimental/hotswap/intent-bundle.js');
|
|
98
|
-
return intentBundleModulePromise;
|
|
99
|
-
}
|
|
98
|
+
import { getQKNormOnesBuffer } from './attention/types.js';
|
|
99
|
+
import {
|
|
100
|
+
getWeightBuffer as getPipelineWeightBuffer,
|
|
101
|
+
getNormWeightBuffer as getPipelineNormWeightBuffer,
|
|
102
|
+
} from './weights.js';
|
|
100
103
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
104
|
+
// Extracted standalone helpers
|
|
105
|
+
import {
|
|
106
|
+
resolvePromptInput,
|
|
107
|
+
releasePerLayerInputBuffer,
|
|
108
|
+
normalizePrefixEmbeddingOverride,
|
|
109
|
+
resolvePrefillEmbeddingInputIds,
|
|
110
|
+
resolvePrefillMultimodalBidirectionalSpan,
|
|
111
|
+
applyPrefixEmbeddingOverride,
|
|
112
|
+
resolvePrefixEmbeddingOverrideTransitionDeclaredBy,
|
|
113
|
+
shouldDisablePrefillCommandBatching,
|
|
114
|
+
resolveEffectivePrefillTokenChunkSize,
|
|
115
|
+
} from './generator-prefill-helpers.js';
|
|
116
|
+
import {
|
|
117
|
+
shouldDisableBatchDecodeAfterShortBatch,
|
|
118
|
+
resolveHotVocabularyBatchDecodeAvailability,
|
|
119
|
+
primePleDecodeRuntimeCache,
|
|
120
|
+
recordPrefillProfileStep,
|
|
121
|
+
resolveTokenText,
|
|
122
|
+
usesReplayPrefillDecode,
|
|
123
|
+
assertIncrementalDecodeSupport,
|
|
124
|
+
summarizeExecutionPlan,
|
|
125
|
+
shouldRetryWithFinitenessFallback,
|
|
126
|
+
createUnhandledFinitenessPolicyError,
|
|
127
|
+
resolveTargetPlanKVDtype,
|
|
128
|
+
resolveCurrentKVCacheDtype,
|
|
129
|
+
cloneRuntimeInferenceWithKVDtype,
|
|
130
|
+
} from './generator-decode-policy.js';
|
|
131
|
+
|
|
132
|
+
const SPECIAL_LIKE_TOKEN_RE = /^(<pad>|<unused\d*>|<eos>|<bos>|<s>|<\/s>|\[PAD\]|\[UNK\]|\[SEP\]|\[CLS\]|<[^>\n]{1,32}>)$/i;
|
|
133
|
+
const FINITENESS_RESET_WORDS = new Uint32Array(4);
|
|
134
|
+
const tokenizerSuppressionCache = new WeakMap();
|
|
135
|
+
const PREFILL_CHUNK_SUBMIT_MODES = new Set(['sync', 'async']);
|
|
136
|
+
|
|
137
|
+
export function resolvePrefillChunkSubmitMode(runtimeConfig, modelConfig) {
|
|
138
|
+
const runtimeSubmit = runtimeConfig?.inference?.session?.prefillChunkSubmitMode;
|
|
139
|
+
const manifestSubmit = modelConfig?.sessionSettings?.prefillChunkSubmitMode;
|
|
140
|
+
const submit = (runtimeSubmit !== undefined && runtimeSubmit !== null)
|
|
141
|
+
? runtimeSubmit
|
|
142
|
+
: manifestSubmit;
|
|
143
|
+
if (submit === undefined || submit === null) {
|
|
144
|
+
throw new Error('[Pipeline] runtime.inference.session.prefillChunkSubmitMode is required.');
|
|
145
|
+
}
|
|
146
|
+
if (!PREFILL_CHUNK_SUBMIT_MODES.has(submit)) {
|
|
147
|
+
throw new Error(
|
|
148
|
+
`[Pipeline] runtime.inference.session.prefillChunkSubmitMode must be "sync" or "async"; got "${String(submit)}".`
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
return submit;
|
|
106
152
|
}
|
|
107
153
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
154
|
+
function getTokenizerSuppressionCache(tokenizer) {
|
|
155
|
+
let cache = tokenizerSuppressionCache.get(tokenizer);
|
|
156
|
+
if (!cache) {
|
|
157
|
+
cache = new Map();
|
|
158
|
+
tokenizerSuppressionCache.set(tokenizer, cache);
|
|
159
|
+
}
|
|
160
|
+
return cache;
|
|
114
161
|
}
|
|
115
162
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
tokenId,
|
|
120
|
-
}) {
|
|
121
|
-
if (hasRangeBackedPerLayerInputs !== true) {
|
|
122
|
-
return false;
|
|
163
|
+
function decodeSingleTokenForSuppression(tokenizer, tokenId) {
|
|
164
|
+
if (!tokenizer || typeof tokenizer.decode !== 'function') {
|
|
165
|
+
return '';
|
|
123
166
|
}
|
|
124
|
-
|
|
125
|
-
return false;
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
if (!Number.isInteger(sentinelIndex)) {
|
|
129
|
-
return false;
|
|
130
|
-
}
|
|
131
|
-
if (!Number.isInteger(tokenId)) {
|
|
132
|
-
return false;
|
|
167
|
+
try {
|
|
168
|
+
return tokenizer.decode([tokenId], false, false);
|
|
169
|
+
} catch {
|
|
170
|
+
return '';
|
|
133
171
|
}
|
|
134
|
-
const hotTokenIndex = pleHotVocabularyRuntime.hotTokenIndexMap?.[tokenId] ?? sentinelIndex;
|
|
135
|
-
return hotTokenIndex !== sentinelIndex;
|
|
136
172
|
}
|
|
137
173
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
)
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
config: state.modelConfig,
|
|
169
|
-
weights: state.weights,
|
|
170
|
-
weightConfig: {
|
|
171
|
-
rmsNormWeightOffset: state.modelConfig?.rmsNormWeightOffset ?? false,
|
|
172
|
-
},
|
|
173
|
-
debugFlags: state.debugFlags,
|
|
174
|
-
});
|
|
174
|
+
function collectTokenizerSuppressedTokenIds(tokenizer, vocabSize, samplingConfig) {
|
|
175
|
+
if (!tokenizer || !Number.isInteger(vocabSize) || vocabSize < 1) {
|
|
176
|
+
return [];
|
|
177
|
+
}
|
|
178
|
+
if (samplingConfig.suppressSpecialTokens !== true && samplingConfig.suppressSpecialLikeTokens !== true) {
|
|
179
|
+
return [];
|
|
180
|
+
}
|
|
181
|
+
const cache = getTokenizerSuppressionCache(tokenizer);
|
|
182
|
+
const cacheKey = [
|
|
183
|
+
vocabSize,
|
|
184
|
+
samplingConfig.suppressSpecialTokens === true ? 'special' : 'plain',
|
|
185
|
+
samplingConfig.suppressSpecialLikeTokens === true ? 'specialLike' : 'literal',
|
|
186
|
+
].join(':');
|
|
187
|
+
const cached = cache.get(cacheKey);
|
|
188
|
+
if (cached) {
|
|
189
|
+
return cached;
|
|
190
|
+
}
|
|
191
|
+
const tokenIds = [];
|
|
192
|
+
for (let tokenId = 0; tokenId < vocabSize; tokenId++) {
|
|
193
|
+
const tokenizerSpecial = samplingConfig.suppressSpecialTokens === true
|
|
194
|
+
&& typeof tokenizer.isSpecialToken === 'function'
|
|
195
|
+
&& tokenizer.isSpecialToken(tokenId);
|
|
196
|
+
const specialLike = samplingConfig.suppressSpecialLikeTokens === true
|
|
197
|
+
&& SPECIAL_LIKE_TOKEN_RE.test(decodeSingleTokenForSuppression(tokenizer, tokenId).trim());
|
|
198
|
+
if (tokenizerSpecial || specialLike) {
|
|
199
|
+
tokenIds.push(tokenId);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
cache.set(cacheKey, tokenIds);
|
|
203
|
+
return tokenIds;
|
|
175
204
|
}
|
|
176
205
|
|
|
177
|
-
function
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
206
|
+
function resolveSuppressedSamplingTokenIds(state, samplingConfig) {
|
|
207
|
+
const vocabSize = state.modelConfig?.vocabSize;
|
|
208
|
+
const suppressed = new Set(samplingConfig.suppressTokenIds);
|
|
209
|
+
const stopTokenIds = new Set(state.modelConfig?.stopTokenIds ?? []);
|
|
210
|
+
const eosToken = state.tokenizer?.getSpecialTokens?.()?.eos;
|
|
211
|
+
if (Number.isInteger(eosToken)) {
|
|
212
|
+
stopTokenIds.add(eosToken);
|
|
181
213
|
}
|
|
182
|
-
state.
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
|
|
186
|
-
const chatOptions = state.modelConfig.chatTemplateThinking === true ? { thinking: true } : undefined;
|
|
187
|
-
if (typeof prompt === 'string') {
|
|
188
|
-
if (useChatTemplate && state.modelConfig.chatTemplateType) {
|
|
189
|
-
if (state.modelConfig.chatTemplateType === 'translategemma') {
|
|
190
|
-
throw new Error(
|
|
191
|
-
`[Pipeline] ${contextLabel}: translategemma chat template requires structured messages. ` +
|
|
192
|
-
'Pass { messages: [...] } instead of a plain string prompt.'
|
|
193
|
-
);
|
|
194
|
-
}
|
|
195
|
-
return applyChatTemplate(prompt, state.modelConfig.chatTemplateType, chatOptions);
|
|
214
|
+
for (const tokenId of collectTokenizerSuppressedTokenIds(state.tokenizer, vocabSize, samplingConfig)) {
|
|
215
|
+
if (!stopTokenIds.has(tokenId)) {
|
|
216
|
+
suppressed.add(tokenId);
|
|
196
217
|
}
|
|
197
|
-
return prompt;
|
|
198
218
|
}
|
|
199
|
-
|
|
200
|
-
if (prompt != null && typeof prompt === 'object' && !Array.isArray(prompt) && 'messages' in prompt && !Array.isArray(prompt.messages)) {
|
|
201
|
-
throw new Error(
|
|
202
|
-
`[Pipeline] ${contextLabel}: prompt.messages must be an array of chat messages, got ${typeof prompt.messages}. ` +
|
|
203
|
-
'Pass { messages: [{ role: "user", content: "..." }, ...] }.'
|
|
204
|
-
);
|
|
205
|
-
}
|
|
206
|
-
const messages = isStructuredChatRequest(prompt)
|
|
207
|
-
? prompt.messages
|
|
208
|
-
: (Array.isArray(prompt) ? prompt : null);
|
|
209
|
-
if (!messages) {
|
|
210
|
-
throw new Error(
|
|
211
|
-
`[Pipeline] ${contextLabel}: prompt must be a string, chat message array, or { messages: [...] }.`
|
|
212
|
-
);
|
|
213
|
-
}
|
|
214
|
-
const templateType = useChatTemplate ? state.modelConfig.chatTemplateType : null;
|
|
215
|
-
return formatChatMessages(messages, templateType, chatOptions);
|
|
219
|
+
return [...suppressed];
|
|
216
220
|
}
|
|
217
221
|
|
|
218
|
-
function
|
|
219
|
-
if (!buffer) {
|
|
222
|
+
async function traceActivationHealth(label, buffer, dtype, elementCount) {
|
|
223
|
+
if (!isTraceEnabled('logits') || !isGpuBufferInstance(buffer)) {
|
|
220
224
|
return;
|
|
221
225
|
}
|
|
222
|
-
const
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
}
|
|
226
|
-
const cachedPleBuffer = pleCache?.ownedBuffers instanceof Set && pleCache.ownedBuffers.has(buffer);
|
|
227
|
-
if (cachedPleBuffer) {
|
|
228
|
-
return;
|
|
229
|
-
}
|
|
230
|
-
if (recorder) {
|
|
231
|
-
recorder.trackTemporaryBuffer(buffer);
|
|
232
|
-
return;
|
|
233
|
-
}
|
|
234
|
-
releaseBuffer(buffer);
|
|
226
|
+
const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
|
|
227
|
+
const data = await readBuffer(buffer, elementCount * bytesPerElement);
|
|
228
|
+
trace.logits(label, getLogitsHealth(decodeReadback(data, dtype)));
|
|
235
229
|
}
|
|
236
230
|
|
|
237
|
-
function
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
}
|
|
241
|
-
if (typeof override !== 'object') {
|
|
242
|
-
throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides must be an object when provided.`);
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
const prefixLength = Number(override.prefixLength);
|
|
246
|
-
if (!Number.isFinite(prefixLength) || prefixLength < 0 || Math.floor(prefixLength) !== prefixLength) {
|
|
247
|
-
throw new Error(
|
|
248
|
-
`[Pipeline] ${contextLabel}: embeddingOverrides.prefixLength must be a non-negative integer.`
|
|
249
|
-
);
|
|
250
|
-
}
|
|
251
|
-
if (prefixLength === 0) {
|
|
252
|
-
return null;
|
|
253
|
-
}
|
|
231
|
+
function ownsBorrowedWeightBuffer(weight) {
|
|
232
|
+
return !isGpuBufferInstance(weight) && !isWeightBuffer(weight);
|
|
233
|
+
}
|
|
254
234
|
|
|
255
|
-
|
|
256
|
-
if (!
|
|
257
|
-
throw new Error(
|
|
258
|
-
`[Pipeline] ${contextLabel}: embeddingOverrides.offset must be a non-negative integer.`
|
|
259
|
-
);
|
|
235
|
+
function borrowLinearWeight(weight, label) {
|
|
236
|
+
if (!weight) {
|
|
237
|
+
throw new Error(`DiffusionGemma self-conditioning missing ${label}.`);
|
|
260
238
|
}
|
|
261
|
-
if (
|
|
239
|
+
if (isSplitWeightBuffer(weight)) {
|
|
262
240
|
throw new Error(
|
|
263
|
-
`
|
|
241
|
+
`DiffusionGemma self-conditioning does not support split weight storage for ${label}.`
|
|
264
242
|
);
|
|
265
243
|
}
|
|
244
|
+
return {
|
|
245
|
+
value: getPipelineWeightBuffer(weight, label),
|
|
246
|
+
owned: ownsBorrowedWeightBuffer(weight),
|
|
247
|
+
};
|
|
248
|
+
}
|
|
266
249
|
|
|
267
|
-
|
|
268
|
-
if (!
|
|
269
|
-
throw new Error(`
|
|
250
|
+
function borrowNormWeight(weight, label) {
|
|
251
|
+
if (!weight) {
|
|
252
|
+
throw new Error(`DiffusionGemma self-conditioning missing ${label}.`);
|
|
270
253
|
}
|
|
271
|
-
|
|
272
|
-
const expectedLength = prefixLength * hiddenSize;
|
|
273
|
-
if (embeddings instanceof Float32Array) {
|
|
274
|
-
if (embeddings.length !== expectedLength) {
|
|
275
|
-
throw new Error(
|
|
276
|
-
`[Pipeline] ${contextLabel}: embedding override length mismatch ` +
|
|
277
|
-
`(expected=${expectedLength}, got=${embeddings.length}).`
|
|
278
|
-
);
|
|
279
|
-
}
|
|
280
|
-
} else if (isGpuBufferInstance(embeddings)) {
|
|
281
|
-
const expectedBytes = expectedLength * Float32Array.BYTES_PER_ELEMENT;
|
|
282
|
-
if (embeddings.size < expectedBytes) {
|
|
283
|
-
throw new Error(
|
|
284
|
-
`[Pipeline] ${contextLabel}: embedding override GPUBuffer too small ` +
|
|
285
|
-
`(expected>=${expectedBytes} bytes, got=${embeddings.size}).`
|
|
286
|
-
);
|
|
287
|
-
}
|
|
288
|
-
} else {
|
|
254
|
+
if (isSplitWeightBuffer(weight)) {
|
|
289
255
|
throw new Error(
|
|
290
|
-
`
|
|
256
|
+
`DiffusionGemma self-conditioning does not support split norm storage for ${label}.`
|
|
291
257
|
);
|
|
292
258
|
}
|
|
293
|
-
|
|
294
259
|
return {
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
embeddings,
|
|
298
|
-
expectedLength,
|
|
299
|
-
byteLength: expectedLength * Float32Array.BYTES_PER_ELEMENT,
|
|
300
|
-
byteOffset: offset * hiddenSize * Float32Array.BYTES_PER_ELEMENT,
|
|
260
|
+
value: getPipelineNormWeightBuffer(weight, label),
|
|
261
|
+
owned: ownsBorrowedWeightBuffer(weight),
|
|
301
262
|
};
|
|
302
263
|
}
|
|
303
264
|
|
|
304
|
-
function
|
|
305
|
-
if (
|
|
306
|
-
return
|
|
307
|
-
}
|
|
308
|
-
if (typeof embeddingInputSpan !== 'object') {
|
|
309
|
-
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan must be an object when provided.`);
|
|
265
|
+
function releaseBorrowedWeight(borrowed) {
|
|
266
|
+
if (!borrowed?.owned) {
|
|
267
|
+
return;
|
|
310
268
|
}
|
|
269
|
+
const value = borrowed.value;
|
|
270
|
+
releaseBuffer(isWeightBuffer(value) ? value.buffer : value);
|
|
271
|
+
}
|
|
311
272
|
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
273
|
+
function canUseChunkedSoftEmbeddingLogits(logitsState, embeddingWeight, embeddingTranspose) {
|
|
274
|
+
return logitsState != null
|
|
275
|
+
&& isWeightBuffer(embeddingWeight)
|
|
276
|
+
&& getWeightDtype(embeddingWeight) === 'f16'
|
|
277
|
+
&& getLayout(embeddingWeight) === 'row'
|
|
278
|
+
&& embeddingTranspose !== true;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function resolveDiffusionGemmaSoftEmbeddingChunkRows(runtimeConfig) {
|
|
282
|
+
const value = runtimeConfig?.inference?.diffusionGemma?.softEmbeddingLogitsChunkRows;
|
|
283
|
+
if (value == null) {
|
|
284
|
+
return undefined;
|
|
323
285
|
}
|
|
324
|
-
if (
|
|
286
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
325
287
|
throw new Error(
|
|
326
|
-
|
|
327
|
-
`exceeds inputIds length ${inputIds.length}.`
|
|
288
|
+
'runtime.inference.diffusionGemma.softEmbeddingLogitsChunkRows must be a positive integer.'
|
|
328
289
|
);
|
|
329
290
|
}
|
|
330
|
-
|
|
331
|
-
replacedInputIds.fill(tokenId, offset, offset + length);
|
|
332
|
-
return replacedInputIds;
|
|
291
|
+
return value;
|
|
333
292
|
}
|
|
334
293
|
|
|
335
|
-
function
|
|
336
|
-
if (
|
|
337
|
-
|
|
338
|
-
}
|
|
339
|
-
if (typeof bidirectionalSpan !== 'object') {
|
|
340
|
-
throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan must be an object when provided.`);
|
|
294
|
+
function normalizeCanvasTokenIds(canvas, context) {
|
|
295
|
+
if (!Array.isArray(canvas) && !ArrayBuffer.isView(canvas)) {
|
|
296
|
+
throw new Error(`[DiffusionGemma] ${context}.canvas must be an array or typed array of token IDs.`);
|
|
341
297
|
}
|
|
298
|
+
return Array.from(canvas, (value, index) => {
|
|
299
|
+
if (!Number.isFinite(value) || Math.floor(value) !== value || value < 0) {
|
|
300
|
+
throw new Error(`[DiffusionGemma] ${context}.canvas[${index}] must be a non-negative integer token ID.`);
|
|
301
|
+
}
|
|
302
|
+
return value;
|
|
303
|
+
});
|
|
304
|
+
}
|
|
342
305
|
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.offset must be a non-negative integer.`);
|
|
306
|
+
function normalizeSelfConditioningLogits(logits, canvasLength, vocabSize) {
|
|
307
|
+
if (logits == null) {
|
|
308
|
+
return null;
|
|
347
309
|
}
|
|
348
|
-
|
|
349
|
-
|
|
310
|
+
const expected = canvasLength * vocabSize;
|
|
311
|
+
const values = logits instanceof Float32Array
|
|
312
|
+
? logits
|
|
313
|
+
: (Array.isArray(logits) ? Float32Array.from(logits) : null);
|
|
314
|
+
if (!values) {
|
|
315
|
+
throw new Error('[DiffusionGemma] selfConditioningLogits must be a Float32Array or number array.');
|
|
350
316
|
}
|
|
351
|
-
if (
|
|
317
|
+
if (values.length !== expected) {
|
|
352
318
|
throw new Error(
|
|
353
|
-
`[
|
|
354
|
-
`exceeds inputIds length ${inputIds.length}.`
|
|
319
|
+
`[DiffusionGemma] selfConditioningLogits length mismatch: expected ${expected}, got ${values.length}.`
|
|
355
320
|
);
|
|
356
321
|
}
|
|
357
|
-
return
|
|
322
|
+
return values;
|
|
358
323
|
}
|
|
359
324
|
|
|
360
|
-
|
|
361
|
-
if (!
|
|
362
|
-
return
|
|
325
|
+
function normalizeSelfConditioningLogitsState(logits, canvasLength, vocabSize) {
|
|
326
|
+
if (logits == null || !isGpuBufferInstance(logits.logitsBuffer)) {
|
|
327
|
+
return null;
|
|
328
|
+
}
|
|
329
|
+
const dtype = logits.logitsDtype;
|
|
330
|
+
if (dtype !== 'f32') {
|
|
331
|
+
throw new Error(`[DiffusionGemma] GPU selfConditioningLogits require f32 logits, got "${dtype}".`);
|
|
363
332
|
}
|
|
364
|
-
if (
|
|
333
|
+
if (logits.vocabSize !== vocabSize) {
|
|
365
334
|
throw new Error(
|
|
366
|
-
`[
|
|
335
|
+
`[DiffusionGemma] GPU selfConditioningLogits vocab mismatch: expected ${vocabSize}, got ${logits.vocabSize}.`
|
|
367
336
|
);
|
|
368
337
|
}
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
const outputBuffer = acquireBuffer(baseTensor.buffer.size, undefined, 'prefill_embedding_override');
|
|
376
|
-
const targetElementOffset = override.offset * hiddenSize;
|
|
377
|
-
const targetByteOffset = targetElementOffset * (baseTensor.dtype === 'f16' ? 2 : 4);
|
|
378
|
-
try {
|
|
379
|
-
const encoder = device.createCommandEncoder();
|
|
380
|
-
encoder.copyBufferToBuffer(baseTensor.buffer, 0, outputBuffer, 0, baseTensor.buffer.size);
|
|
381
|
-
if (baseTensor.dtype === 'f32') {
|
|
382
|
-
if (isGpuBufferInstance(override.embeddings)) {
|
|
383
|
-
encoder.copyBufferToBuffer(override.embeddings, 0, outputBuffer, override.byteOffset, override.byteLength);
|
|
384
|
-
device.queue.submit([encoder.finish()]);
|
|
385
|
-
} else {
|
|
386
|
-
device.queue.submit([encoder.finish()]);
|
|
387
|
-
uploadData(outputBuffer, override.embeddings, override.byteOffset);
|
|
388
|
-
}
|
|
389
|
-
return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
if (isGpuBufferInstance(override.embeddings)) {
|
|
393
|
-
const overrideTensor = createTensor(
|
|
394
|
-
override.embeddings,
|
|
395
|
-
'f32',
|
|
396
|
-
[override.prefixLength, hiddenSize],
|
|
397
|
-
'prefill_embedding_override_f32'
|
|
398
|
-
);
|
|
399
|
-
assertImplicitDtypeTransitionAllowed({
|
|
400
|
-
executionPolicies,
|
|
401
|
-
fromDtype: 'f32',
|
|
402
|
-
toDtype: 'f16',
|
|
403
|
-
op: 'prefill_embedding_override',
|
|
404
|
-
detail: 'Prefix embedding override would pack GPU-provided f32 features into an f16 activation buffer.',
|
|
405
|
-
});
|
|
406
|
-
const castedOverride = await castF32ToF16(overrideTensor);
|
|
407
|
-
try {
|
|
408
|
-
encoder.copyBufferToBuffer(
|
|
409
|
-
castedOverride.buffer,
|
|
410
|
-
0,
|
|
411
|
-
outputBuffer,
|
|
412
|
-
targetByteOffset,
|
|
413
|
-
castedOverride.buffer.size
|
|
414
|
-
);
|
|
415
|
-
device.queue.submit([encoder.finish()]);
|
|
416
|
-
} finally {
|
|
417
|
-
releaseBuffer(castedOverride.buffer);
|
|
418
|
-
}
|
|
419
|
-
} else {
|
|
420
|
-
assertImplicitDtypeTransitionAllowed({
|
|
421
|
-
executionPolicies,
|
|
422
|
-
fromDtype: 'f32',
|
|
423
|
-
toDtype: 'f16',
|
|
424
|
-
op: 'prefill_embedding_override',
|
|
425
|
-
detail: 'Prefix embedding override would pack CPU-provided f32 features into an f16 activation buffer.',
|
|
426
|
-
});
|
|
427
|
-
const packedOverride = f32ToF16Array(override.embeddings);
|
|
428
|
-
device.queue.submit([encoder.finish()]);
|
|
429
|
-
uploadData(outputBuffer, packedOverride, targetByteOffset);
|
|
430
|
-
}
|
|
431
|
-
return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
|
|
432
|
-
} catch (error) {
|
|
433
|
-
releaseBuffer(outputBuffer);
|
|
434
|
-
throw error;
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
/**
|
|
439
|
-
* Resolve display text for a token ID sequence.
|
|
440
|
-
*
|
|
441
|
-
* Preference order:
|
|
442
|
-
* 1. renderTokenText (custom primary renderer) — if provided and returns non-empty string.
|
|
443
|
-
* 2. tokenizer.decode (default primary) — if renderTokenText is not provided.
|
|
444
|
-
* 3. renderFallbackTokenText (custom fallback) — if provided and returns non-empty string,
|
|
445
|
-
* unless primary returned empty string and fallback looks like a special token (<...>),
|
|
446
|
-
* in which case empty string is preserved to keep skip-special behavior deterministic.
|
|
447
|
-
* 4. tokenizer.decode with skipSpecialTokens=false (default fallback) — same special-token guard.
|
|
448
|
-
* 5. fallbackText — static fallback string (default '?').
|
|
449
|
-
*/
|
|
450
|
-
function resolveTokenText(tokenizer, tokenIds, fallbackText = '?', renderTokenText, renderFallbackTokenText) {
|
|
451
|
-
const renderPrimary = typeof renderTokenText === 'function'
|
|
452
|
-
? renderTokenText
|
|
453
|
-
: (ids) => tokenizer?.decode?.(ids);
|
|
454
|
-
const renderFallback = typeof renderFallbackTokenText === 'function'
|
|
455
|
-
? renderFallbackTokenText
|
|
456
|
-
: (ids) => tokenizer?.decode?.(ids, false);
|
|
457
|
-
|
|
458
|
-
const primaryText = renderPrimary(tokenIds);
|
|
459
|
-
if (typeof primaryText === 'string' && primaryText.length > 0) {
|
|
460
|
-
return primaryText;
|
|
461
|
-
}
|
|
462
|
-
|
|
463
|
-
const fallback = renderFallback(tokenIds);
|
|
464
|
-
if (typeof fallback === 'string' && fallback.length > 0) {
|
|
465
|
-
// Keep skip-special behavior deterministic: if primary decoding filtered this
|
|
466
|
-
// token to empty, do not reintroduce obvious special-token text via fallback.
|
|
467
|
-
if (
|
|
468
|
-
primaryText === ''
|
|
469
|
-
&& /^<[^>\n]{1,80}>$/.test(fallback.trim())
|
|
470
|
-
) {
|
|
471
|
-
return '';
|
|
472
|
-
}
|
|
473
|
-
return fallback;
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
return fallbackText;
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
function usesReplayPrefillDecode(state) {
|
|
480
|
-
return state?.modelConfig?.decodeStrategy === 'replay_prefill';
|
|
481
|
-
}
|
|
482
|
-
|
|
483
|
-
export function shouldDisablePrefillCommandBatching(state, opts, multimodalBidirectionalSpan) {
|
|
484
|
-
if (
|
|
485
|
-
opts?.disableCommandBatching === true
|
|
486
|
-
|| opts?.debug === true
|
|
487
|
-
|| (Array.isArray(opts?.debugLayers) && opts.debugLayers.length > 0)
|
|
488
|
-
) {
|
|
489
|
-
return true;
|
|
490
|
-
}
|
|
491
|
-
if (state?.kvCache?.layout === 'bdpa_paged') {
|
|
492
|
-
return true;
|
|
493
|
-
}
|
|
494
|
-
if (multimodalBidirectionalSpan == null) {
|
|
495
|
-
return false;
|
|
496
|
-
}
|
|
497
|
-
if (state?.kvCache?.hasGPUCache?.() !== true) {
|
|
498
|
-
return false;
|
|
338
|
+
if (logits.canvasLength !== canvasLength) {
|
|
339
|
+
throw new Error(
|
|
340
|
+
`[DiffusionGemma] GPU selfConditioningLogits canvas mismatch: expected ${canvasLength}, got ${logits.canvasLength}.`
|
|
341
|
+
);
|
|
499
342
|
}
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
return true;
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
function assertIncrementalDecodeSupport(state, operation) {
|
|
507
|
-
if (!usesReplayPrefillDecode(state)) {
|
|
508
|
-
return;
|
|
343
|
+
const temperature = logits.temperature ?? 1.0;
|
|
344
|
+
if (typeof temperature !== 'number' || !Number.isFinite(temperature) || temperature <= 0) {
|
|
345
|
+
throw new Error('[DiffusionGemma] GPU selfConditioningLogits temperature must be positive.');
|
|
509
346
|
}
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
347
|
+
return {
|
|
348
|
+
logitsBuffer: logits.logitsBuffer,
|
|
349
|
+
logitsDtype: dtype,
|
|
350
|
+
temperature,
|
|
351
|
+
releaseOnUse: logits.releaseOnUse !== false,
|
|
352
|
+
};
|
|
515
353
|
}
|
|
516
354
|
|
|
517
|
-
function
|
|
518
|
-
if (
|
|
355
|
+
function normalizeSelfConditioningSoftEmbeddingState(state, canvasLength, hiddenSize) {
|
|
356
|
+
if (state?.kind !== 'soft_embedding') {
|
|
519
357
|
return null;
|
|
520
358
|
}
|
|
521
|
-
if (
|
|
522
|
-
|
|
523
|
-
return null;
|
|
359
|
+
if (!isGpuBufferInstance(state.buffer)) {
|
|
360
|
+
throw new Error('[DiffusionGemma] GPU selfConditioning soft embedding requires a GPU buffer.');
|
|
524
361
|
}
|
|
525
|
-
|
|
526
|
-
|
|
362
|
+
const dtype = state.dtype ?? 'f32';
|
|
363
|
+
if (dtype !== 'f32') {
|
|
364
|
+
throw new Error(`[DiffusionGemma] GPU selfConditioning soft embedding requires f32 dtype, got "${dtype}".`);
|
|
527
365
|
}
|
|
528
|
-
if (
|
|
529
|
-
|
|
366
|
+
if (state.canvasLength !== canvasLength) {
|
|
367
|
+
throw new Error(
|
|
368
|
+
`[DiffusionGemma] GPU selfConditioning soft embedding canvas mismatch: ` +
|
|
369
|
+
`expected ${canvasLength}, got ${state.canvasLength}.`
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
if (state.hiddenSize !== hiddenSize) {
|
|
373
|
+
throw new Error(
|
|
374
|
+
`[DiffusionGemma] GPU selfConditioning soft embedding hidden mismatch: ` +
|
|
375
|
+
`expected ${hiddenSize}, got ${state.hiddenSize}.`
|
|
376
|
+
);
|
|
530
377
|
}
|
|
531
378
|
return {
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
readbackInterval: plan.readbackInterval ?? null,
|
|
537
|
-
batchSize: plan.defaultBatchSize,
|
|
538
|
-
stopCheckMode: plan.defaultStopCheckMode,
|
|
539
|
-
ringTokens: plan.ringTokens ?? null,
|
|
540
|
-
ringStop: plan.ringStop ?? null,
|
|
541
|
-
ringStaging: plan.ringStaging ?? null,
|
|
379
|
+
buffer: state.buffer,
|
|
380
|
+
dtype,
|
|
381
|
+
releaseOnUse: state.releaseOnUse !== false,
|
|
382
|
+
scaled: state.scaled === true,
|
|
542
383
|
};
|
|
543
384
|
}
|
|
544
385
|
|
|
545
|
-
|
|
546
|
-
if (error?.name === 'FinitenessError') {
|
|
547
|
-
return true;
|
|
548
|
-
}
|
|
549
|
-
const message = typeof error?.message === 'string'
|
|
550
|
-
? error.message
|
|
551
|
-
: (typeof error === 'string' ? error : '');
|
|
552
|
-
if (!message.startsWith('[Sampling]')) {
|
|
553
|
-
return false;
|
|
554
|
-
}
|
|
555
|
-
return message.includes('no finite candidate logits after masking the pad token')
|
|
556
|
-
|| message.includes('Softmax produced no finite candidate probabilities');
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
function createUnhandledFinitenessPolicyError(state, contextLabel, error) {
|
|
560
|
-
const activePlan = resolveActiveExecutionPlan(state);
|
|
561
|
-
const wrapped = new Error(
|
|
562
|
-
`[Pipeline] ${contextLabel}: finiteness guard triggered for kernelPath ` +
|
|
563
|
-
`"${activePlan.kernelPathId ?? 'none'}" under fail-fast policy. ` +
|
|
564
|
-
'Resolve the unstable path with an explicit capability-aware execution override, ' +
|
|
565
|
-
'or opt into alternate-plan recovery with ' +
|
|
566
|
-
'runtime.inference.compute.rangeAwareSelectiveWidening.onTrigger="fallback-plan".',
|
|
567
|
-
error instanceof Error ? { cause: error } : undefined
|
|
568
|
-
);
|
|
569
|
-
wrapped.name = error?.name === 'FinitenessError' ? error.name : 'FinitenessError';
|
|
570
|
-
return wrapped;
|
|
571
|
-
}
|
|
572
|
-
|
|
573
|
-
function assertResolvedKVDtype(kvDtype, contextLabel) {
|
|
574
|
-
if (kvDtype === 'f16' || kvDtype === 'f32') {
|
|
575
|
-
return kvDtype;
|
|
576
|
-
}
|
|
577
|
-
throw new Error(
|
|
578
|
-
`[Pipeline] ${contextLabel}: expected execution-plan kvDtype to resolve to "f16" or "f32", ` +
|
|
579
|
-
`got ${JSON.stringify(kvDtype)}.`
|
|
580
|
-
);
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
function resolveTargetPlanKVDtype(plan, contextLabel) {
|
|
584
|
-
return assertResolvedKVDtype(
|
|
585
|
-
plan?.kernelPath?.kvDtype ?? plan?.activationDtype ?? null,
|
|
586
|
-
contextLabel
|
|
587
|
-
);
|
|
588
|
-
}
|
|
386
|
+
let intentBundleModulePromise = null;
|
|
589
387
|
|
|
590
|
-
function
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
?? plan?.kernelPath?.kvDtype
|
|
594
|
-
?? state?.runtimeConfig?.inference?.session?.kvcache?.kvDtype
|
|
595
|
-
?? plan?.activationDtype
|
|
596
|
-
?? null,
|
|
597
|
-
contextLabel
|
|
598
|
-
);
|
|
388
|
+
async function getExperimentalIntentBundleModule() {
|
|
389
|
+
intentBundleModulePromise ??= import('../../../experimental/hotswap/intent-bundle.js');
|
|
390
|
+
return intentBundleModulePromise;
|
|
599
391
|
}
|
|
600
392
|
|
|
601
|
-
function cloneRuntimeInferenceWithKVDtype(state, kvDtype) {
|
|
602
|
-
const runtimeInference = state?.runtimeConfig?.inference;
|
|
603
|
-
if (!runtimeInference?.session?.kvcache) {
|
|
604
|
-
throw new Error(
|
|
605
|
-
'[Pipeline] runtime.inference.session.kvcache is required for finiteness fallback KV-cache recovery.'
|
|
606
|
-
);
|
|
607
|
-
}
|
|
608
|
-
return mergeRuntimeValues(runtimeInference, {
|
|
609
|
-
session: {
|
|
610
|
-
kvcache: {
|
|
611
|
-
kvDtype,
|
|
612
|
-
},
|
|
613
|
-
},
|
|
614
|
-
});
|
|
615
|
-
}
|
|
616
393
|
|
|
617
394
|
export class PipelineGenerator {
|
|
618
395
|
|
|
@@ -751,8 +528,14 @@ export class PipelineGenerator {
|
|
|
751
528
|
this.#state.stats.ttftMs = 0;
|
|
752
529
|
this.#state.stats.decodeTimeMs = 0;
|
|
753
530
|
this.#state.stats.decodeRecordMs = 0;
|
|
531
|
+
this.#state.stats.decodeRecordOps = 0;
|
|
532
|
+
this.#state.stats.decodeRecordPasses = 0;
|
|
533
|
+
this.#state.stats.decodeRecordOpLabels = {};
|
|
754
534
|
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
755
535
|
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
536
|
+
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
537
|
+
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
538
|
+
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
756
539
|
this.#state.decodeStepCount = 0;
|
|
757
540
|
this.#state.disableRecordedLogits = false;
|
|
758
541
|
this.#state.disableFusedDecode = false;
|
|
@@ -762,6 +545,12 @@ export class PipelineGenerator {
|
|
|
762
545
|
totalBatchedTimeMs: 0,
|
|
763
546
|
totalUnbatchedTimeMs: 0,
|
|
764
547
|
gpuSubmissions: 0,
|
|
548
|
+
requestedBatchTokens: 0,
|
|
549
|
+
effectiveBatchTokens: 0,
|
|
550
|
+
executedBatchTokens: 0,
|
|
551
|
+
resolvedBatchTokens: 0,
|
|
552
|
+
maxBatchTokenCap: null,
|
|
553
|
+
batchClampCount: 0,
|
|
765
554
|
};
|
|
766
555
|
resetActiveExecutionPlan(this.#state);
|
|
767
556
|
this.#state.decodeRing?.reset();
|
|
@@ -812,14 +601,96 @@ export class PipelineGenerator {
|
|
|
812
601
|
_sampleNextTokenFromLogits(logits, generatedIds, opts) {
|
|
813
602
|
const sampledLogits = Float32Array.from(logits);
|
|
814
603
|
applyRepetitionPenalty(sampledLogits, generatedIds, opts.repetitionPenalty);
|
|
604
|
+
// Optional pre-sample logit mask. Callers pass `opts.logitMaskFn` to
|
|
605
|
+
// implement grammar/schema-constrained decoding. The hook receives the
|
|
606
|
+
// mutable logit buffer (after repetition penalty) plus the running token
|
|
607
|
+
// sequence so it can track parse state across decode steps.
|
|
608
|
+
if (typeof opts?.logitMaskFn === "function") {
|
|
609
|
+
try {
|
|
610
|
+
opts.logitMaskFn(sampledLogits, {
|
|
611
|
+
generatedIds,
|
|
612
|
+
tokenizer: this.#state.tokenizer ?? null,
|
|
613
|
+
vocabSize: this.#state.modelConfig?.vocabSize ?? sampledLogits.length,
|
|
614
|
+
});
|
|
615
|
+
} catch (maskError) {
|
|
616
|
+
log.warn("Pipeline", `logitMaskFn threw; continuing without mask: ${maskError}`);
|
|
617
|
+
}
|
|
618
|
+
}
|
|
815
619
|
const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
|
|
816
|
-
|
|
620
|
+
const tokenId = sample(sampledLogits, {
|
|
817
621
|
temperature: opts.temperature,
|
|
818
622
|
topP: opts.topP,
|
|
819
623
|
topK: opts.topK,
|
|
820
624
|
padTokenId,
|
|
821
625
|
seed: opts.seed,
|
|
626
|
+
suppressTokenIds: opts.suppressTokenIds,
|
|
822
627
|
});
|
|
628
|
+
if (typeof opts.onLogits === 'function') {
|
|
629
|
+
opts.onLogits(sampledLogits, {
|
|
630
|
+
tokenId,
|
|
631
|
+
inputTokenCount: Array.isArray(generatedIds) ? generatedIds.length : null,
|
|
632
|
+
});
|
|
633
|
+
}
|
|
634
|
+
return tokenId;
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
_resolvePrefillTokenChunkSize(inputIds) {
|
|
638
|
+
const chunkSize = resolveEffectivePrefillTokenChunkSize(this.#state);
|
|
639
|
+
if (chunkSize === undefined) {
|
|
640
|
+
throw new Error('inference.session.prefillTokenChunkSize is required; use null to disable token-chunked prefill.');
|
|
641
|
+
}
|
|
642
|
+
if (chunkSize === null) {
|
|
643
|
+
return null;
|
|
644
|
+
}
|
|
645
|
+
if (!Number.isInteger(chunkSize) || chunkSize <= 0) {
|
|
646
|
+
throw new Error('inference.session.prefillTokenChunkSize must be null or a positive integer.');
|
|
647
|
+
}
|
|
648
|
+
return chunkSize < inputIds.length ? chunkSize : null;
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
async _commitPrefillHiddenChunk(prefillResult) {
|
|
652
|
+
const {
|
|
653
|
+
numTokens,
|
|
654
|
+
startPos,
|
|
655
|
+
currentRecorder,
|
|
656
|
+
recordProfile,
|
|
657
|
+
currentHiddenBuffer,
|
|
658
|
+
} = prefillResult;
|
|
659
|
+
|
|
660
|
+
try {
|
|
661
|
+
if (currentRecorder) {
|
|
662
|
+
await currentRecorder.submitAndWait();
|
|
663
|
+
await recordProfile(currentRecorder);
|
|
664
|
+
} else {
|
|
665
|
+
const device = getDevice();
|
|
666
|
+
if (device) {
|
|
667
|
+
await device.queue.onSubmittedWorkDone();
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
this.#state.currentSeqLen = startPos + numTokens;
|
|
671
|
+
} finally {
|
|
672
|
+
releaseBuffer(currentHiddenBuffer);
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
async _prefillInputIdsToLogits(inputIds, opts) {
|
|
677
|
+
const chunkSize = this._resolvePrefillTokenChunkSize(inputIds);
|
|
678
|
+
if (chunkSize === null) {
|
|
679
|
+
return this._prefill(inputIds, opts);
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
for (let offset = 0; offset < inputIds.length; offset += chunkSize) {
|
|
683
|
+
const end = Math.min(offset + chunkSize, inputIds.length);
|
|
684
|
+
const chunk = inputIds.slice(offset, end);
|
|
685
|
+
const isFinalChunk = end === inputIds.length;
|
|
686
|
+
if (isFinalChunk) {
|
|
687
|
+
return this._prefill(chunk, opts);
|
|
688
|
+
}
|
|
689
|
+
const prefillResult = await this._prefillToHidden(chunk, opts);
|
|
690
|
+
await this._commitPrefillHiddenChunk(prefillResult);
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
return this._prefill(inputIds, opts);
|
|
823
694
|
}
|
|
824
695
|
|
|
825
696
|
async _prefillPromptToLogits(prompt, opts, contextLabel) {
|
|
@@ -830,8 +701,9 @@ export class PipelineGenerator {
|
|
|
830
701
|
}
|
|
831
702
|
|
|
832
703
|
let logits;
|
|
704
|
+
const runPrefill = () => this._prefillInputIdsToLogits(inputIds, opts);
|
|
833
705
|
try {
|
|
834
|
-
logits = await
|
|
706
|
+
logits = await runPrefill();
|
|
835
707
|
} catch (error) {
|
|
836
708
|
if (!this._shouldUseFinitenessFallback(error, contextLabel)) {
|
|
837
709
|
throw error;
|
|
@@ -841,7 +713,7 @@ export class PipelineGenerator {
|
|
|
841
713
|
opts,
|
|
842
714
|
contextLabel,
|
|
843
715
|
opts.maxTokens ?? 1,
|
|
844
|
-
|
|
716
|
+
runPrefill,
|
|
845
717
|
prefillStartSeqLen
|
|
846
718
|
);
|
|
847
719
|
}
|
|
@@ -880,6 +752,11 @@ export class PipelineGenerator {
|
|
|
880
752
|
return this._matchesStopSequence(generatedIds, runtime.stopSequenceStart, opts.stopSequences);
|
|
881
753
|
}
|
|
882
754
|
|
|
755
|
+
_recordStopReason(reason, tokenId = null) {
|
|
756
|
+
this.#state.stats.stopReason = reason;
|
|
757
|
+
this.#state.stats.stopTokenId = Number.isInteger(tokenId) ? tokenId : null;
|
|
758
|
+
}
|
|
759
|
+
|
|
883
760
|
async *_generateTokensInternal(prompt, options = {}, mode = 'text') {
|
|
884
761
|
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
885
762
|
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
@@ -891,10 +768,21 @@ export class PipelineGenerator {
|
|
|
891
768
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
892
769
|
this.#state.stats.gpuTimeDecodeMs = undefined;
|
|
893
770
|
this.#state.stats.decodeRecordMs = 0;
|
|
771
|
+
this.#state.stats.decodeRecordOps = 0;
|
|
772
|
+
this.#state.stats.decodeRecordPasses = 0;
|
|
773
|
+
this.#state.stats.decodeRecordOpLabels = {};
|
|
894
774
|
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
895
775
|
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
776
|
+
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
777
|
+
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
778
|
+
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
779
|
+
this.#state.stats.prefillRecordMs = 0;
|
|
780
|
+
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
896
781
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
897
782
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
783
|
+
this.#state.stats.singleTokenReadbackMapWaitMs = 0;
|
|
784
|
+
this.#state.stats.singleTokenReadbackCleanupMs = 0;
|
|
785
|
+
this.#state.stats.singleTokenReadbackCopyMs = 0;
|
|
898
786
|
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
899
787
|
this.#state.stats.pleHotVocabularyHits = 0;
|
|
900
788
|
this.#state.stats.pleHotVocabularyMisses = 0;
|
|
@@ -904,19 +792,25 @@ export class PipelineGenerator {
|
|
|
904
792
|
this.#state.stats.plePreparedTokenCacheBytes = 0;
|
|
905
793
|
this.#state.stats.decodeMode = null;
|
|
906
794
|
this.#state.stats.batchGuardReason = null;
|
|
795
|
+
this.#state.stats.stopReason = null;
|
|
796
|
+
this.#state.stats.stopTokenId = null;
|
|
907
797
|
this.#state.stats.ttftMs = 0;
|
|
908
798
|
const startTime = performance.now();
|
|
909
799
|
|
|
910
800
|
const opts = resolveGenerateOptions(this.#state, options);
|
|
801
|
+
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
802
|
+
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
911
803
|
// Validate and normalize sampling parameters through single source of truth
|
|
912
804
|
const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
|
|
913
805
|
opts.temperature = samplingConfig.temperature;
|
|
914
806
|
opts.topP = samplingConfig.topP;
|
|
915
807
|
opts.topK = samplingConfig.topK;
|
|
916
808
|
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
809
|
+
opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this.#state, samplingConfig);
|
|
917
810
|
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
918
811
|
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
919
|
-
|
|
812
|
+
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
813
|
+
if (diagnosticsEnabled || tsirFixtureCfg) {
|
|
920
814
|
const captureConfig = {
|
|
921
815
|
...createDefaultCaptureConfig(),
|
|
922
816
|
enabled: true,
|
|
@@ -925,13 +819,20 @@ export class PipelineGenerator {
|
|
|
925
819
|
};
|
|
926
820
|
validateCaptureConfig(captureConfig);
|
|
927
821
|
this.#state.operatorDiagnostics = {
|
|
928
|
-
enabled: true,
|
|
822
|
+
enabled: diagnosticsEnabled === true,
|
|
929
823
|
captureConfig,
|
|
930
|
-
emitter: new OperatorEventEmitter({
|
|
824
|
+
emitter: diagnosticsEnabled ? new OperatorEventEmitter({
|
|
931
825
|
modelHash: this.#state.manifest?.modelId ?? null,
|
|
932
826
|
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
933
827
|
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
934
|
-
}),
|
|
828
|
+
}) : null,
|
|
829
|
+
tsirFixture: tsirFixtureCfg ? {
|
|
830
|
+
dir: tsirFixtureCfg.dir,
|
|
831
|
+
layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
|
|
832
|
+
? tsirFixtureCfg.layerFilter
|
|
833
|
+
: null,
|
|
834
|
+
records: [],
|
|
835
|
+
} : null,
|
|
935
836
|
};
|
|
936
837
|
}
|
|
937
838
|
const activePlan = opts.executionPlan ?? resolveActiveExecutionPlan(this.#state);
|
|
@@ -1051,6 +952,7 @@ export class PipelineGenerator {
|
|
|
1051
952
|
yield* emitToken(this, firstToken, decodeToken);
|
|
1052
953
|
|
|
1053
954
|
if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
|
|
955
|
+
this._recordStopReason('stop-token-or-sequence', firstToken);
|
|
1054
956
|
this.#state.stats.decodeTimeMs = 0;
|
|
1055
957
|
this.#state.stats.tokensGenerated = 1;
|
|
1056
958
|
this.#state.stats.decodeTokens = 1;
|
|
@@ -1277,6 +1179,599 @@ export class PipelineGenerator {
|
|
|
1277
1179
|
// Generation Public API
|
|
1278
1180
|
// ==========================================================================
|
|
1279
1181
|
|
|
1182
|
+
/*
|
|
1183
|
+
* Truncate the KV cache back to `seqLen` tokens and set `currentSeqLen` to
|
|
1184
|
+
* match. Intended for "prefix-reuse" workflows where a caller wants to run
|
|
1185
|
+
* several decodes that share a common prompt prefix: prefill once with the
|
|
1186
|
+
* shared prefix, decode the first tail, then `resetToSeqLen(prefixLen)` to
|
|
1187
|
+
* drop the tail's KV entries and reuse the prefix KV for the next tail.
|
|
1188
|
+
*
|
|
1189
|
+
* Only valid when no decode is in progress.
|
|
1190
|
+
*/
|
|
1191
|
+
resetToSeqLen(seqLen) {
|
|
1192
|
+
if (this.#state.isGenerating) {
|
|
1193
|
+
throw new Error('InferencePipeline.resetToSeqLen: cannot reset while generation is in progress');
|
|
1194
|
+
}
|
|
1195
|
+
const target = Math.max(0, Math.floor(Number(seqLen) || 0));
|
|
1196
|
+
if (!Number.isFinite(target)) {
|
|
1197
|
+
throw new Error('InferencePipeline.resetToSeqLen: seqLen must be a finite non-negative integer');
|
|
1198
|
+
}
|
|
1199
|
+
if (target > this.#state.currentSeqLen) {
|
|
1200
|
+
throw new Error(
|
|
1201
|
+
`InferencePipeline.resetToSeqLen: target ${target} exceeds currentSeqLen ${this.#state.currentSeqLen}`
|
|
1202
|
+
);
|
|
1203
|
+
}
|
|
1204
|
+
this.#state.kvCache?.truncate?.(target);
|
|
1205
|
+
this.#state.currentSeqLen = target;
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
async _createDiffusionGemmaSelfConditioningEmbeddings(canvasIds, selfConditioningLogits, opts) {
|
|
1209
|
+
const config = this.#state.modelConfig;
|
|
1210
|
+
const canvasLength = canvasIds.length;
|
|
1211
|
+
const hiddenSize = config.hiddenSize;
|
|
1212
|
+
const vocabSize = config.vocabSize;
|
|
1213
|
+
const elementCount = canvasLength * hiddenSize;
|
|
1214
|
+
const softEmbeddingState = normalizeSelfConditioningSoftEmbeddingState(
|
|
1215
|
+
selfConditioningLogits,
|
|
1216
|
+
canvasLength,
|
|
1217
|
+
hiddenSize
|
|
1218
|
+
);
|
|
1219
|
+
const logitsState = softEmbeddingState
|
|
1220
|
+
? null
|
|
1221
|
+
: normalizeSelfConditioningLogitsState(selfConditioningLogits, canvasLength, vocabSize);
|
|
1222
|
+
const logits = (softEmbeddingState || logitsState)
|
|
1223
|
+
? null
|
|
1224
|
+
: normalizeSelfConditioningLogits(selfConditioningLogits, canvasLength, vocabSize);
|
|
1225
|
+
const weights = this.#state.weights.get('diffusion_gemma_self_conditioning');
|
|
1226
|
+
if (!weights || typeof weights !== 'object') {
|
|
1227
|
+
throw new Error(
|
|
1228
|
+
'DiffusionGemma self-conditioning weights were not loaded. ' +
|
|
1229
|
+
'Expected model.decoder.self_conditioning tensors in the manifest.'
|
|
1230
|
+
);
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
const embedBufferRaw = this.#state.weights.get('embed');
|
|
1234
|
+
if (!embedBufferRaw) {
|
|
1235
|
+
throw new Error('DiffusionGemma self-conditioning requires loaded embed_tokens weights.');
|
|
1236
|
+
}
|
|
1237
|
+
const borrowed = {
|
|
1238
|
+
preNorm: null,
|
|
1239
|
+
gateProj: null,
|
|
1240
|
+
upProj: null,
|
|
1241
|
+
downProj: null,
|
|
1242
|
+
softEmbedding: null,
|
|
1243
|
+
};
|
|
1244
|
+
let softLogitsTensor = null;
|
|
1245
|
+
let softLogitsOwned = true;
|
|
1246
|
+
let softEmbeddingStateTensor = null;
|
|
1247
|
+
let softEmbeddingStateOwned = false;
|
|
1248
|
+
let softmaxTensor = null;
|
|
1249
|
+
let softEmbeddings = null;
|
|
1250
|
+
let scaledSoftEmbeddings = null;
|
|
1251
|
+
let baseEmbeddings = null;
|
|
1252
|
+
let normedSoft = null;
|
|
1253
|
+
let gate = null;
|
|
1254
|
+
let up = null;
|
|
1255
|
+
let activated = null;
|
|
1256
|
+
let down = null;
|
|
1257
|
+
let combined = null;
|
|
1258
|
+
let output = null;
|
|
1259
|
+
|
|
1260
|
+
const releaseTensorOnce = (() => {
|
|
1261
|
+
const released = new Set();
|
|
1262
|
+
return (tensor) => {
|
|
1263
|
+
const buffer = tensor?.buffer ?? null;
|
|
1264
|
+
if (!buffer || released.has(buffer)) return;
|
|
1265
|
+
released.add(buffer);
|
|
1266
|
+
releaseBuffer(buffer);
|
|
1267
|
+
};
|
|
1268
|
+
})();
|
|
1269
|
+
|
|
1270
|
+
try {
|
|
1271
|
+
borrowed.preNorm = borrowNormWeight(weights.preNorm, 'diffusion_gemma_self_conditioning.pre_norm');
|
|
1272
|
+
borrowed.gateProj = borrowLinearWeight(weights.gateProj, 'diffusion_gemma_self_conditioning.gate_proj');
|
|
1273
|
+
borrowed.upProj = borrowLinearWeight(weights.upProj, 'diffusion_gemma_self_conditioning.up_proj');
|
|
1274
|
+
borrowed.downProj = borrowLinearWeight(weights.downProj, 'diffusion_gemma_self_conditioning.down_proj');
|
|
1275
|
+
if (isSplitWeightBuffer(embedBufferRaw) && this.#state.embeddingTranspose === true) {
|
|
1276
|
+
throw new Error(
|
|
1277
|
+
'DiffusionGemma self-conditioning split embeddings require row-major embedding storage.'
|
|
1278
|
+
);
|
|
1279
|
+
}
|
|
1280
|
+
borrowed.softEmbedding = isSplitWeightBuffer(embedBufferRaw)
|
|
1281
|
+
? null
|
|
1282
|
+
: borrowLinearWeight(embedBufferRaw, 'diffusion_gemma_self_conditioning.embed_tokens');
|
|
1283
|
+
|
|
1284
|
+
const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
|
|
1285
|
+
const embedDtype = isCpuWeightBuffer(embedBufferRaw)
|
|
1286
|
+
? embedBufferRaw.dtype
|
|
1287
|
+
: getWeightDtype(embedBufferRaw);
|
|
1288
|
+
const embedMetadata = getWeightMetadata(embedBufferRaw);
|
|
1289
|
+
baseEmbeddings = await embed(canvasIds, embedBuffer, {
|
|
1290
|
+
hiddenSize,
|
|
1291
|
+
vocabSize,
|
|
1292
|
+
scaleEmbeddings: config.scaleEmbeddings,
|
|
1293
|
+
debug: opts.debug,
|
|
1294
|
+
recorder: null,
|
|
1295
|
+
transpose: this.#state.embeddingTranspose,
|
|
1296
|
+
activationDtype: 'f32',
|
|
1297
|
+
embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
|
|
1298
|
+
embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
|
|
1299
|
+
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1300
|
+
operatorDiagnostics: this.#state.operatorDiagnostics,
|
|
1301
|
+
});
|
|
1302
|
+
|
|
1303
|
+
if (softEmbeddingState) {
|
|
1304
|
+
softEmbeddingStateOwned = softEmbeddingState.releaseOnUse;
|
|
1305
|
+
softEmbeddings = createTensor(
|
|
1306
|
+
softEmbeddingState.buffer,
|
|
1307
|
+
softEmbeddingState.dtype,
|
|
1308
|
+
[canvasLength, hiddenSize],
|
|
1309
|
+
'diffusion_gemma_self_conditioning_soft_embedding'
|
|
1310
|
+
);
|
|
1311
|
+
softEmbeddingStateTensor = softEmbeddings;
|
|
1312
|
+
if (!softEmbeddingState.scaled) {
|
|
1313
|
+
scaledSoftEmbeddings = await runScale(softEmbeddings, Math.sqrt(hiddenSize), {
|
|
1314
|
+
count: elementCount,
|
|
1315
|
+
});
|
|
1316
|
+
if (softEmbeddingStateOwned) {
|
|
1317
|
+
releaseTensorOnce(softEmbeddings);
|
|
1318
|
+
}
|
|
1319
|
+
softEmbeddings = scaledSoftEmbeddings;
|
|
1320
|
+
softEmbeddingStateTensor = null;
|
|
1321
|
+
scaledSoftEmbeddings = null;
|
|
1322
|
+
}
|
|
1323
|
+
} else if (logitsState || logits) {
|
|
1324
|
+
const softmaxTemperature = logitsState?.temperature ?? 1.0;
|
|
1325
|
+
if (logitsState) {
|
|
1326
|
+
softLogitsOwned = logitsState.releaseOnUse;
|
|
1327
|
+
softLogitsTensor = createTensor(
|
|
1328
|
+
logitsState.logitsBuffer,
|
|
1329
|
+
logitsState.logitsDtype,
|
|
1330
|
+
[canvasLength, vocabSize],
|
|
1331
|
+
'diffusion_gemma_self_conditioning_logits'
|
|
1332
|
+
);
|
|
1333
|
+
} else {
|
|
1334
|
+
const logitsBuffer = acquireBuffer(logits.byteLength, undefined, 'diffusion_gemma_self_conditioning_logits');
|
|
1335
|
+
uploadData(logitsBuffer, logits);
|
|
1336
|
+
softLogitsTensor = createTensor(logitsBuffer, 'f32', [canvasLength, vocabSize], 'diffusion_gemma_self_conditioning_logits');
|
|
1337
|
+
}
|
|
1338
|
+
if (canUseChunkedSoftEmbeddingLogits(
|
|
1339
|
+
logitsState,
|
|
1340
|
+
borrowed.softEmbedding?.value,
|
|
1341
|
+
this.#state.embeddingTranspose
|
|
1342
|
+
)) {
|
|
1343
|
+
softEmbeddings = await runSoftEmbeddingLogitsF16(
|
|
1344
|
+
softLogitsTensor,
|
|
1345
|
+
borrowed.softEmbedding.value,
|
|
1346
|
+
canvasLength,
|
|
1347
|
+
hiddenSize,
|
|
1348
|
+
vocabSize,
|
|
1349
|
+
{
|
|
1350
|
+
temperature: softmaxTemperature,
|
|
1351
|
+
chunkRows: resolveDiffusionGemmaSoftEmbeddingChunkRows(this.#state.runtimeConfig),
|
|
1352
|
+
}
|
|
1353
|
+
);
|
|
1354
|
+
} else {
|
|
1355
|
+
softmaxTensor = await runSoftmax(softLogitsTensor, -1, {
|
|
1356
|
+
batchSize: canvasLength,
|
|
1357
|
+
size: vocabSize,
|
|
1358
|
+
temperature: softmaxTemperature,
|
|
1359
|
+
});
|
|
1360
|
+
softEmbeddings = isSplitWeightBuffer(embedBufferRaw)
|
|
1361
|
+
? await runSoftEmbeddingSplitF16(
|
|
1362
|
+
softmaxTensor,
|
|
1363
|
+
embedBufferRaw,
|
|
1364
|
+
canvasLength,
|
|
1365
|
+
hiddenSize,
|
|
1366
|
+
vocabSize
|
|
1367
|
+
)
|
|
1368
|
+
: await runMatmul(
|
|
1369
|
+
softmaxTensor,
|
|
1370
|
+
borrowed.softEmbedding.value,
|
|
1371
|
+
canvasLength,
|
|
1372
|
+
hiddenSize,
|
|
1373
|
+
vocabSize,
|
|
1374
|
+
{
|
|
1375
|
+
transposeB: this.#state.embeddingTranspose === true,
|
|
1376
|
+
role: 'diffusion_gemma_self_conditioning_embed',
|
|
1377
|
+
outputDtype: 'f32',
|
|
1378
|
+
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1379
|
+
}
|
|
1380
|
+
);
|
|
1381
|
+
}
|
|
1382
|
+
scaledSoftEmbeddings = await runScale(softEmbeddings, Math.sqrt(hiddenSize), {
|
|
1383
|
+
count: elementCount,
|
|
1384
|
+
});
|
|
1385
|
+
if (softEmbeddings !== softEmbeddingStateTensor || softEmbeddingStateOwned) {
|
|
1386
|
+
releaseTensorOnce(softEmbeddings);
|
|
1387
|
+
}
|
|
1388
|
+
softEmbeddings = scaledSoftEmbeddings;
|
|
1389
|
+
scaledSoftEmbeddings = null;
|
|
1390
|
+
} else {
|
|
1391
|
+
const zeroBuffer = acquireBuffer(elementCount * Float32Array.BYTES_PER_ELEMENT, undefined, 'diffusion_gemma_self_conditioning_zero');
|
|
1392
|
+
uploadData(zeroBuffer, new Uint8Array(elementCount * Float32Array.BYTES_PER_ELEMENT));
|
|
1393
|
+
softEmbeddings = createTensor(zeroBuffer, 'f32', [canvasLength, hiddenSize], 'diffusion_gemma_self_conditioning_zero');
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
normedSoft = await runRMSNorm(softEmbeddings, borrowed.preNorm.value, config.rmsNormEps, {
|
|
1397
|
+
batchSize: canvasLength,
|
|
1398
|
+
hiddenSize,
|
|
1399
|
+
rmsNormWeightOffset: false,
|
|
1400
|
+
});
|
|
1401
|
+
const intermediateSize = config.intermediateSize;
|
|
1402
|
+
gate = await runMatmul(
|
|
1403
|
+
normedSoft,
|
|
1404
|
+
borrowed.gateProj.value,
|
|
1405
|
+
canvasLength,
|
|
1406
|
+
intermediateSize,
|
|
1407
|
+
hiddenSize,
|
|
1408
|
+
{
|
|
1409
|
+
transposeB: 'auto',
|
|
1410
|
+
role: 'diffusion_gemma_self_conditioning_gate',
|
|
1411
|
+
outputDtype: 'f32',
|
|
1412
|
+
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1413
|
+
}
|
|
1414
|
+
);
|
|
1415
|
+
up = await runMatmul(
|
|
1416
|
+
normedSoft,
|
|
1417
|
+
borrowed.upProj.value,
|
|
1418
|
+
canvasLength,
|
|
1419
|
+
intermediateSize,
|
|
1420
|
+
hiddenSize,
|
|
1421
|
+
{
|
|
1422
|
+
transposeB: 'auto',
|
|
1423
|
+
role: 'diffusion_gemma_self_conditioning_up',
|
|
1424
|
+
outputDtype: 'f32',
|
|
1425
|
+
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1426
|
+
}
|
|
1427
|
+
);
|
|
1428
|
+
activated = await runGeLU(up, {
|
|
1429
|
+
size: canvasLength * intermediateSize,
|
|
1430
|
+
gate,
|
|
1431
|
+
});
|
|
1432
|
+
down = await runMatmul(
|
|
1433
|
+
activated,
|
|
1434
|
+
borrowed.downProj.value,
|
|
1435
|
+
canvasLength,
|
|
1436
|
+
hiddenSize,
|
|
1437
|
+
intermediateSize,
|
|
1438
|
+
{
|
|
1439
|
+
transposeB: 'auto',
|
|
1440
|
+
role: 'diffusion_gemma_self_conditioning_down',
|
|
1441
|
+
outputDtype: 'f32',
|
|
1442
|
+
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1443
|
+
}
|
|
1444
|
+
);
|
|
1445
|
+
combined = await runResidualAdd(baseEmbeddings, down, elementCount, {
|
|
1446
|
+
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1447
|
+
});
|
|
1448
|
+
const postNormWeight = weights.postNorm
|
|
1449
|
+
? borrowNormWeight(weights.postNorm, 'diffusion_gemma_self_conditioning.post_norm')
|
|
1450
|
+
: {
|
|
1451
|
+
value: getQKNormOnesBuffer(hiddenSize),
|
|
1452
|
+
owned: false,
|
|
1453
|
+
};
|
|
1454
|
+
borrowed.postNorm = postNormWeight;
|
|
1455
|
+
output = await runRMSNorm(combined, postNormWeight.value, config.rmsNormEps, {
|
|
1456
|
+
batchSize: canvasLength,
|
|
1457
|
+
hiddenSize,
|
|
1458
|
+
rmsNormWeightOffset: false,
|
|
1459
|
+
});
|
|
1460
|
+
return output;
|
|
1461
|
+
} catch (error) {
|
|
1462
|
+
releaseTensorOnce(output);
|
|
1463
|
+
throw error;
|
|
1464
|
+
} finally {
|
|
1465
|
+
releaseTensorOnce(combined);
|
|
1466
|
+
releaseTensorOnce(down);
|
|
1467
|
+
releaseTensorOnce(activated);
|
|
1468
|
+
releaseTensorOnce(up);
|
|
1469
|
+
releaseTensorOnce(gate);
|
|
1470
|
+
releaseTensorOnce(normedSoft);
|
|
1471
|
+
if (softEmbeddings !== softEmbeddingStateTensor || softEmbeddingStateOwned) {
|
|
1472
|
+
releaseTensorOnce(softEmbeddings);
|
|
1473
|
+
}
|
|
1474
|
+
releaseTensorOnce(softmaxTensor);
|
|
1475
|
+
if (softLogitsOwned) {
|
|
1476
|
+
releaseTensorOnce(softLogitsTensor);
|
|
1477
|
+
}
|
|
1478
|
+
releaseTensorOnce(baseEmbeddings);
|
|
1479
|
+
releaseBorrowedWeight(borrowed.postNorm);
|
|
1480
|
+
releaseBorrowedWeight(borrowed.downProj);
|
|
1481
|
+
releaseBorrowedWeight(borrowed.upProj);
|
|
1482
|
+
releaseBorrowedWeight(borrowed.gateProj);
|
|
1483
|
+
releaseBorrowedWeight(borrowed.preNorm);
|
|
1484
|
+
releaseBorrowedWeight(borrowed.softEmbedding);
|
|
1485
|
+
}
|
|
1486
|
+
}
|
|
1487
|
+
|
|
1488
|
+
async _createDiffusionGemmaSelfConditioningSoftEmbeddingState(logitsState, canvasLength, hiddenSize, vocabSize) {
|
|
1489
|
+
const embedBufferRaw = this.#state.weights.get('embed');
|
|
1490
|
+
if (!canUseChunkedSoftEmbeddingLogits(
|
|
1491
|
+
logitsState,
|
|
1492
|
+
embedBufferRaw,
|
|
1493
|
+
this.#state.embeddingTranspose
|
|
1494
|
+
)) {
|
|
1495
|
+
return null;
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
const borrowed = borrowLinearWeight(embedBufferRaw, 'diffusion_gemma_self_conditioning.embed_tokens');
|
|
1499
|
+
let softEmbedding = null;
|
|
1500
|
+
let scaledSoftEmbedding = null;
|
|
1501
|
+
try {
|
|
1502
|
+
const logitsTensor = createTensor(
|
|
1503
|
+
logitsState.logitsBuffer,
|
|
1504
|
+
logitsState.logitsDtype,
|
|
1505
|
+
[canvasLength, vocabSize],
|
|
1506
|
+
'diffusion_gemma_self_conditioning_logits'
|
|
1507
|
+
);
|
|
1508
|
+
softEmbedding = await runSoftEmbeddingLogitsF16(
|
|
1509
|
+
logitsTensor,
|
|
1510
|
+
borrowed.value,
|
|
1511
|
+
canvasLength,
|
|
1512
|
+
hiddenSize,
|
|
1513
|
+
vocabSize,
|
|
1514
|
+
{
|
|
1515
|
+
temperature: logitsState.temperature,
|
|
1516
|
+
chunkRows: resolveDiffusionGemmaSoftEmbeddingChunkRows(this.#state.runtimeConfig),
|
|
1517
|
+
}
|
|
1518
|
+
);
|
|
1519
|
+
scaledSoftEmbedding = await runScale(softEmbedding, Math.sqrt(hiddenSize), {
|
|
1520
|
+
count: canvasLength * hiddenSize,
|
|
1521
|
+
});
|
|
1522
|
+
const returnedBuffer = scaledSoftEmbedding.buffer;
|
|
1523
|
+
return {
|
|
1524
|
+
kind: 'soft_embedding',
|
|
1525
|
+
buffer: returnedBuffer,
|
|
1526
|
+
dtype: scaledSoftEmbedding.dtype,
|
|
1527
|
+
canvasLength,
|
|
1528
|
+
hiddenSize,
|
|
1529
|
+
scaled: true,
|
|
1530
|
+
releaseOnUse: true,
|
|
1531
|
+
release() {
|
|
1532
|
+
releaseBuffer(returnedBuffer);
|
|
1533
|
+
},
|
|
1534
|
+
};
|
|
1535
|
+
} finally {
|
|
1536
|
+
if (softEmbedding?.buffer && softEmbedding.buffer !== scaledSoftEmbedding?.buffer) {
|
|
1537
|
+
releaseBuffer(softEmbedding.buffer);
|
|
1538
|
+
}
|
|
1539
|
+
releaseBorrowedWeight(borrowed);
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
|
|
1543
|
+
async computeDiffusionGemmaCanvasLogits(args, options = {}) {
|
|
1544
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
1545
|
+
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
1546
|
+
throw new Error('Generation already in progress');
|
|
1547
|
+
}
|
|
1548
|
+
const canvasIds = normalizeCanvasTokenIds(args?.canvas, 'computeDiffusionGemmaCanvasLogits');
|
|
1549
|
+
if (canvasIds.length === 0) {
|
|
1550
|
+
throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasLogits requires at least one canvas token.');
|
|
1551
|
+
}
|
|
1552
|
+
this._assertTokenIdsInRange(canvasIds, 'computeDiffusionGemmaCanvasLogits.canvas');
|
|
1553
|
+
const seqLenBefore = this.#state.currentSeqLen;
|
|
1554
|
+
const opts = {
|
|
1555
|
+
...resolvePrefillOptions(this.#state, {
|
|
1556
|
+
...options,
|
|
1557
|
+
useChatTemplate: false,
|
|
1558
|
+
}),
|
|
1559
|
+
_diffusionGemmaDecoder: true,
|
|
1560
|
+
};
|
|
1561
|
+
let selfConditioned = null;
|
|
1562
|
+
let currentHiddenBuffer = null;
|
|
1563
|
+
try {
|
|
1564
|
+
selfConditioned = await this._createDiffusionGemmaSelfConditioningEmbeddings(
|
|
1565
|
+
canvasIds,
|
|
1566
|
+
args?.selfConditioningLogits ?? null,
|
|
1567
|
+
opts
|
|
1568
|
+
);
|
|
1569
|
+
const prefillResult = await this._prefillToHidden(canvasIds, {
|
|
1570
|
+
...opts,
|
|
1571
|
+
embeddingOverrides: {
|
|
1572
|
+
offset: 0,
|
|
1573
|
+
prefixLength: canvasIds.length,
|
|
1574
|
+
embeddings: selfConditioned.buffer,
|
|
1575
|
+
},
|
|
1576
|
+
});
|
|
1577
|
+
const {
|
|
1578
|
+
numTokens,
|
|
1579
|
+
currentRecorder,
|
|
1580
|
+
recordProfile,
|
|
1581
|
+
debugCheckBuffer,
|
|
1582
|
+
} = prefillResult;
|
|
1583
|
+
currentHiddenBuffer = prefillResult.currentHiddenBuffer;
|
|
1584
|
+
if (currentRecorder) {
|
|
1585
|
+
await currentRecorder.submitAndWait();
|
|
1586
|
+
await recordProfile(currentRecorder);
|
|
1587
|
+
} else {
|
|
1588
|
+
const device = getDevice();
|
|
1589
|
+
if (device) {
|
|
1590
|
+
await device.queue.onSubmittedWorkDone();
|
|
1591
|
+
}
|
|
1592
|
+
}
|
|
1593
|
+
const logits = await computeLogits(
|
|
1594
|
+
currentHiddenBuffer,
|
|
1595
|
+
numTokens,
|
|
1596
|
+
getLogitsWeights(this.#state),
|
|
1597
|
+
getLogitsConfig(this.#state),
|
|
1598
|
+
this.#state.useGPU,
|
|
1599
|
+
this.#state.debugFlags,
|
|
1600
|
+
undefined,
|
|
1601
|
+
debugCheckBuffer,
|
|
1602
|
+
this.#state.runtimeConfig.shared.debug.probes,
|
|
1603
|
+
{ lastPositionOnly: false },
|
|
1604
|
+
this.#state.operatorDiagnostics
|
|
1605
|
+
);
|
|
1606
|
+
const expected = canvasIds.length * this.#state.modelConfig.vocabSize;
|
|
1607
|
+
if (logits.length !== expected) {
|
|
1608
|
+
throw new Error(
|
|
1609
|
+
`[DiffusionGemma] canvas logits length mismatch: expected ${expected}, got ${logits.length}.`
|
|
1610
|
+
);
|
|
1611
|
+
}
|
|
1612
|
+
return logits;
|
|
1613
|
+
} finally {
|
|
1614
|
+
this.#state.currentSeqLen = seqLenBefore;
|
|
1615
|
+
if (currentHiddenBuffer) {
|
|
1616
|
+
releaseBuffer(currentHiddenBuffer);
|
|
1617
|
+
}
|
|
1618
|
+
if (selfConditioned?.buffer) {
|
|
1619
|
+
releaseBuffer(selfConditioned.buffer);
|
|
1620
|
+
}
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
1623
|
+
|
|
1624
|
+
async computeDiffusionGemmaCanvasStep(args, options = {}) {
|
|
1625
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
1626
|
+
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
1627
|
+
throw new Error('Generation already in progress');
|
|
1628
|
+
}
|
|
1629
|
+
const canvasIds = normalizeCanvasTokenIds(args?.canvas, 'computeDiffusionGemmaCanvasStep');
|
|
1630
|
+
if (canvasIds.length === 0) {
|
|
1631
|
+
throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasStep requires at least one canvas token.');
|
|
1632
|
+
}
|
|
1633
|
+
this._assertTokenIdsInRange(canvasIds, 'computeDiffusionGemmaCanvasStep.canvas');
|
|
1634
|
+
const temperature = args?.temperature;
|
|
1635
|
+
if (typeof temperature !== 'number' || !Number.isFinite(temperature) || temperature <= 0) {
|
|
1636
|
+
throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasStep requires a positive temperature.');
|
|
1637
|
+
}
|
|
1638
|
+
const seqLenBefore = this.#state.currentSeqLen;
|
|
1639
|
+
const opts = {
|
|
1640
|
+
...resolvePrefillOptions(this.#state, {
|
|
1641
|
+
...options,
|
|
1642
|
+
useChatTemplate: false,
|
|
1643
|
+
}),
|
|
1644
|
+
_diffusionGemmaDecoder: true,
|
|
1645
|
+
};
|
|
1646
|
+
let selfConditioned = null;
|
|
1647
|
+
let currentHiddenBuffer = null;
|
|
1648
|
+
let logitsBuffer = null;
|
|
1649
|
+
let statsBuffers = null;
|
|
1650
|
+
try {
|
|
1651
|
+
selfConditioned = await this._createDiffusionGemmaSelfConditioningEmbeddings(
|
|
1652
|
+
canvasIds,
|
|
1653
|
+
args?.selfConditioningLogits ?? null,
|
|
1654
|
+
opts
|
|
1655
|
+
);
|
|
1656
|
+
const prefillResult = await this._prefillToHidden(canvasIds, {
|
|
1657
|
+
...opts,
|
|
1658
|
+
embeddingOverrides: {
|
|
1659
|
+
offset: 0,
|
|
1660
|
+
prefixLength: canvasIds.length,
|
|
1661
|
+
embeddings: selfConditioned.buffer,
|
|
1662
|
+
},
|
|
1663
|
+
});
|
|
1664
|
+
const {
|
|
1665
|
+
numTokens,
|
|
1666
|
+
currentRecorder,
|
|
1667
|
+
recordProfile,
|
|
1668
|
+
debugCheckBuffer,
|
|
1669
|
+
} = prefillResult;
|
|
1670
|
+
currentHiddenBuffer = prefillResult.currentHiddenBuffer;
|
|
1671
|
+
if (currentRecorder) {
|
|
1672
|
+
await currentRecorder.submitAndWait();
|
|
1673
|
+
await recordProfile(currentRecorder);
|
|
1674
|
+
} else {
|
|
1675
|
+
const device = getDevice();
|
|
1676
|
+
if (device) {
|
|
1677
|
+
await device.queue.onSubmittedWorkDone();
|
|
1678
|
+
}
|
|
1679
|
+
}
|
|
1680
|
+
if (selfConditioned?.buffer) {
|
|
1681
|
+
releaseBuffer(selfConditioned.buffer);
|
|
1682
|
+
selfConditioned = null;
|
|
1683
|
+
}
|
|
1684
|
+
|
|
1685
|
+
const logitsResult = await computeLogitsGPU(
|
|
1686
|
+
currentHiddenBuffer,
|
|
1687
|
+
numTokens,
|
|
1688
|
+
getLogitsWeights(this.#state),
|
|
1689
|
+
getLogitsConfig(this.#state),
|
|
1690
|
+
this.#state.debugFlags,
|
|
1691
|
+
this.#state.operatorDiagnostics
|
|
1692
|
+
);
|
|
1693
|
+
if (!logitsResult?.logitsBuffer) {
|
|
1694
|
+
throw new Error('[DiffusionGemma] GPU canvas step requires GPU logits.');
|
|
1695
|
+
}
|
|
1696
|
+
logitsBuffer = logitsResult.logitsBuffer;
|
|
1697
|
+
if (currentHiddenBuffer) {
|
|
1698
|
+
releaseBuffer(currentHiddenBuffer);
|
|
1699
|
+
currentHiddenBuffer = null;
|
|
1700
|
+
}
|
|
1701
|
+
if (logitsResult.logitsDtype !== 'f32') {
|
|
1702
|
+
throw new Error(
|
|
1703
|
+
`[DiffusionGemma] GPU canvas stats require f32 logits, got "${logitsResult.logitsDtype}".`
|
|
1704
|
+
);
|
|
1705
|
+
}
|
|
1706
|
+
if (logitsResult.vocabSize !== this.#state.modelConfig.vocabSize) {
|
|
1707
|
+
throw new Error(
|
|
1708
|
+
`[DiffusionGemma] canvas logits vocab mismatch: expected ${this.#state.modelConfig.vocabSize}, ` +
|
|
1709
|
+
`got ${logitsResult.vocabSize}.`
|
|
1710
|
+
);
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
statsBuffers = await runDiffusionGemmaCanvasStats(logitsBuffer, {
|
|
1714
|
+
canvasLength: canvasIds.length,
|
|
1715
|
+
vocabSize: logitsResult.vocabSize,
|
|
1716
|
+
temperature,
|
|
1717
|
+
padTokenId: this.#state.modelConfig.diffusionGemma?.padTokenId ?? null,
|
|
1718
|
+
logitSoftcap: this.#state.modelConfig.finalLogitSoftcapping ?? 0,
|
|
1719
|
+
});
|
|
1720
|
+
const [argmaxData, entropyData] = await Promise.all([
|
|
1721
|
+
readBuffer(statsBuffers.argmaxBuffer, canvasIds.length * Uint32Array.BYTES_PER_ELEMENT),
|
|
1722
|
+
readBuffer(statsBuffers.entropyBuffer, canvasIds.length * Float32Array.BYTES_PER_ELEMENT),
|
|
1723
|
+
]);
|
|
1724
|
+
const argmaxCanvas = Int32Array.from(new Uint32Array(argmaxData));
|
|
1725
|
+
const entropies = new Float32Array(entropyData);
|
|
1726
|
+
const logitsState = {
|
|
1727
|
+
logitsBuffer,
|
|
1728
|
+
logitsDtype: logitsResult.logitsDtype,
|
|
1729
|
+
vocabSize: logitsResult.vocabSize,
|
|
1730
|
+
canvasLength: canvasIds.length,
|
|
1731
|
+
temperature,
|
|
1732
|
+
releaseOnUse: true,
|
|
1733
|
+
};
|
|
1734
|
+
const selfConditioningState =
|
|
1735
|
+
await this._createDiffusionGemmaSelfConditioningSoftEmbeddingState(
|
|
1736
|
+
logitsState,
|
|
1737
|
+
canvasIds.length,
|
|
1738
|
+
this.#state.modelConfig.hiddenSize,
|
|
1739
|
+
logitsResult.vocabSize
|
|
1740
|
+
);
|
|
1741
|
+
if (selfConditioningState) {
|
|
1742
|
+
releaseBuffer(logitsBuffer);
|
|
1743
|
+
logitsBuffer = null;
|
|
1744
|
+
} else {
|
|
1745
|
+
const returnedLogitsBuffer = logitsBuffer;
|
|
1746
|
+
logitsBuffer = null;
|
|
1747
|
+
logitsState.release = () => {
|
|
1748
|
+
releaseBuffer(returnedLogitsBuffer);
|
|
1749
|
+
};
|
|
1750
|
+
}
|
|
1751
|
+
return {
|
|
1752
|
+
argmaxCanvas,
|
|
1753
|
+
entropies,
|
|
1754
|
+
selfConditioningLogits: selfConditioningState ?? logitsState,
|
|
1755
|
+
};
|
|
1756
|
+
} finally {
|
|
1757
|
+
this.#state.currentSeqLen = seqLenBefore;
|
|
1758
|
+
if (statsBuffers?.argmaxBuffer) {
|
|
1759
|
+
releaseBuffer(statsBuffers.argmaxBuffer);
|
|
1760
|
+
}
|
|
1761
|
+
if (statsBuffers?.entropyBuffer) {
|
|
1762
|
+
releaseBuffer(statsBuffers.entropyBuffer);
|
|
1763
|
+
}
|
|
1764
|
+
if (logitsBuffer) {
|
|
1765
|
+
releaseBuffer(logitsBuffer);
|
|
1766
|
+
}
|
|
1767
|
+
if (currentHiddenBuffer) {
|
|
1768
|
+
releaseBuffer(currentHiddenBuffer);
|
|
1769
|
+
}
|
|
1770
|
+
if (selfConditioned?.buffer) {
|
|
1771
|
+
releaseBuffer(selfConditioned.buffer);
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
1774
|
+
}
|
|
1280
1775
|
|
|
1281
1776
|
async *generate(prompt, options = {}) {
|
|
1282
1777
|
yield* this._generateTokensInternal(prompt, options, 'text');
|
|
@@ -1297,10 +1792,21 @@ export class PipelineGenerator {
|
|
|
1297
1792
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
1298
1793
|
this.#state.stats.gpuTimeDecodeMs = undefined;
|
|
1299
1794
|
this.#state.stats.decodeRecordMs = 0;
|
|
1795
|
+
this.#state.stats.decodeRecordOps = 0;
|
|
1796
|
+
this.#state.stats.decodeRecordPasses = 0;
|
|
1797
|
+
this.#state.stats.decodeRecordOpLabels = {};
|
|
1300
1798
|
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
1301
1799
|
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
1800
|
+
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
1801
|
+
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
1802
|
+
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
1803
|
+
this.#state.stats.prefillRecordMs = 0;
|
|
1804
|
+
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
1302
1805
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
1303
1806
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
1807
|
+
this.#state.stats.singleTokenReadbackMapWaitMs = 0;
|
|
1808
|
+
this.#state.stats.singleTokenReadbackCleanupMs = 0;
|
|
1809
|
+
this.#state.stats.singleTokenReadbackCopyMs = 0;
|
|
1304
1810
|
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
1305
1811
|
this.#state.stats.pleHotVocabularyHits = 0;
|
|
1306
1812
|
this.#state.stats.pleHotVocabularyMisses = 0;
|
|
@@ -1310,18 +1816,23 @@ export class PipelineGenerator {
|
|
|
1310
1816
|
this.#state.stats.plePreparedTokenCacheBytes = 0;
|
|
1311
1817
|
this.#state.stats.decodeMode = null;
|
|
1312
1818
|
this.#state.stats.batchGuardReason = null;
|
|
1819
|
+
this.#state.stats.stopReason = null;
|
|
1820
|
+
this.#state.stats.stopTokenId = null;
|
|
1313
1821
|
this.#state.stats.ttftMs = 0;
|
|
1314
1822
|
const startTime = performance.now();
|
|
1315
1823
|
const opts = resolveGenerateOptions(this.#state, options);
|
|
1824
|
+
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
1316
1825
|
// Validate and normalize sampling parameters through single source of truth
|
|
1317
1826
|
const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
|
|
1318
1827
|
opts.temperature = samplingConfig.temperature;
|
|
1319
1828
|
opts.topP = samplingConfig.topP;
|
|
1320
1829
|
opts.topK = samplingConfig.topK;
|
|
1321
1830
|
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
1831
|
+
opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this.#state, samplingConfig);
|
|
1322
1832
|
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
1323
1833
|
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
1324
|
-
|
|
1834
|
+
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
1835
|
+
if (diagnosticsEnabled || tsirFixtureCfg) {
|
|
1325
1836
|
const captureConfig = {
|
|
1326
1837
|
...createDefaultCaptureConfig(),
|
|
1327
1838
|
enabled: true,
|
|
@@ -1330,13 +1841,20 @@ export class PipelineGenerator {
|
|
|
1330
1841
|
};
|
|
1331
1842
|
validateCaptureConfig(captureConfig);
|
|
1332
1843
|
this.#state.operatorDiagnostics = {
|
|
1333
|
-
enabled: true,
|
|
1844
|
+
enabled: diagnosticsEnabled === true,
|
|
1334
1845
|
captureConfig,
|
|
1335
|
-
emitter: new OperatorEventEmitter({
|
|
1846
|
+
emitter: diagnosticsEnabled ? new OperatorEventEmitter({
|
|
1336
1847
|
modelHash: this.#state.manifest?.modelId ?? null,
|
|
1337
1848
|
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
1338
1849
|
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
1339
|
-
}),
|
|
1850
|
+
}) : null,
|
|
1851
|
+
tsirFixture: tsirFixtureCfg ? {
|
|
1852
|
+
dir: tsirFixtureCfg.dir,
|
|
1853
|
+
layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
|
|
1854
|
+
? tsirFixtureCfg.layerFilter
|
|
1855
|
+
: null,
|
|
1856
|
+
records: [],
|
|
1857
|
+
} : null,
|
|
1340
1858
|
};
|
|
1341
1859
|
}
|
|
1342
1860
|
|
|
@@ -1388,6 +1906,7 @@ export class PipelineGenerator {
|
|
|
1388
1906
|
tokenIds.push(tokenId);
|
|
1389
1907
|
}
|
|
1390
1908
|
} else {
|
|
1909
|
+
this._recordStopReason('stop-token-or-sequence', firstToken);
|
|
1391
1910
|
this.#state.stats.decodeTimeMs = 0;
|
|
1392
1911
|
this.#state.stats.tokensGenerated = 1;
|
|
1393
1912
|
this.#state.stats.decodeTokens = 1;
|
|
@@ -1676,8 +2195,14 @@ export class PipelineGenerator {
|
|
|
1676
2195
|
this.#state.stats.prefillProfileSteps = [];
|
|
1677
2196
|
this.#state.decodeRing?.reset();
|
|
1678
2197
|
this.#state.stats.decodeRecordMs = 0;
|
|
2198
|
+
this.#state.stats.decodeRecordOps = 0;
|
|
2199
|
+
this.#state.stats.decodeRecordPasses = 0;
|
|
2200
|
+
this.#state.stats.decodeRecordOpLabels = {};
|
|
1679
2201
|
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
1680
2202
|
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
2203
|
+
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
2204
|
+
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
2205
|
+
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
1681
2206
|
this.#state.stats.ttftMs = 0;
|
|
1682
2207
|
const startTime = performance.now();
|
|
1683
2208
|
|
|
@@ -1704,6 +2229,7 @@ export class PipelineGenerator {
|
|
|
1704
2229
|
topK: opts.topK,
|
|
1705
2230
|
padTokenId,
|
|
1706
2231
|
seed: opts.seed,
|
|
2232
|
+
suppressTokenIds: opts.suppressTokenIds,
|
|
1707
2233
|
});
|
|
1708
2234
|
|
|
1709
2235
|
const decodeRuntime = {
|
|
@@ -1726,6 +2252,7 @@ export class PipelineGenerator {
|
|
|
1726
2252
|
if (options.onToken) options.onToken(firstToken, firstText);
|
|
1727
2253
|
|
|
1728
2254
|
if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
|
|
2255
|
+
this._recordStopReason('stop-token-or-sequence', firstToken);
|
|
1729
2256
|
this.#state.stats.decodeTimeMs = 0;
|
|
1730
2257
|
this.#state.stats.tokensGenerated = 1;
|
|
1731
2258
|
this.#state.stats.decodeTokens = 1;
|
|
@@ -1785,6 +2312,7 @@ export class PipelineGenerator {
|
|
|
1785
2312
|
const hasLinearLayers = hasLinearAttentionLayers(this.#state.modelConfig.layerTypes);
|
|
1786
2313
|
const replayPrefillDecode = usesReplayPrefillDecode(this.#state);
|
|
1787
2314
|
const gpuSamplingAvailable = isGPUSamplingAvailable() && !hasCpuWeights;
|
|
2315
|
+
const hasSuppressedSamplingTokens = Array.isArray(opts.suppressTokenIds) && opts.suppressTokenIds.length > 0;
|
|
1788
2316
|
const hasRangeBackedPerLayerInputs = hasRangeBackedPerLayerInputEmbeddings({
|
|
1789
2317
|
config: this.#state.modelConfig,
|
|
1790
2318
|
weights: this.#state.weights,
|
|
@@ -1792,6 +2320,7 @@ export class PipelineGenerator {
|
|
|
1792
2320
|
const hasGpuSplitPerLayerInputs = hasGpuSplitPerLayerInputEmbeddings({
|
|
1793
2321
|
config: this.#state.modelConfig,
|
|
1794
2322
|
weights: this.#state.weights,
|
|
2323
|
+
perLayerInputsSession: resolvedPerLayerInputsSession,
|
|
1795
2324
|
});
|
|
1796
2325
|
const pleHotVocabularyRuntime = getPleHotVocabularyRuntime({ weights: this.#state.weights });
|
|
1797
2326
|
const resolveCurrentHotVocabularyBatchDecodeAvailable = () => resolveHotVocabularyBatchDecodeAvailability({
|
|
@@ -1807,7 +2336,7 @@ export class PipelineGenerator {
|
|
|
1807
2336
|
batchSize: executionPlan.batchSize,
|
|
1808
2337
|
useGPU: this.#state.useGPU,
|
|
1809
2338
|
gpuSamplingAvailable,
|
|
1810
|
-
disableMultiTokenDecode: executionPlan.disableMultiTokenDecode,
|
|
2339
|
+
disableMultiTokenDecode: executionPlan.disableMultiTokenDecode || hasSuppressedSamplingTokens,
|
|
1811
2340
|
disableCommandBatching: executionPlan.disableCommandBatching,
|
|
1812
2341
|
isBdpaPagedLayout: this.#state.kvCache?.layout === 'bdpa_paged',
|
|
1813
2342
|
finitenessFallbackWindowOpen: this._hasFinitenessFallbackWindow(),
|
|
@@ -1822,6 +2351,7 @@ export class PipelineGenerator {
|
|
|
1822
2351
|
else if (!this.#state.useGPU) reason = 'no_gpu';
|
|
1823
2352
|
else if (!gpuSamplingAvailable) reason = 'no_gpu_sampling';
|
|
1824
2353
|
else if (executionPlan.disableCommandBatching) reason = 'command_batching_disabled';
|
|
2354
|
+
else if (hasSuppressedSamplingTokens) reason = 'sampling_suppression_requires_cpu_logits';
|
|
1825
2355
|
else if (executionPlan.disableMultiTokenDecode) reason = 'multi_token_decode_disabled';
|
|
1826
2356
|
else if (executionPlan.batchSize <= 1) reason = 'batch_size_1';
|
|
1827
2357
|
else if (this.#state.kvCache?.layout === 'bdpa_paged') reason = 'bdpa_paged_layout';
|
|
@@ -1852,7 +2382,10 @@ export class PipelineGenerator {
|
|
|
1852
2382
|
}
|
|
1853
2383
|
|
|
1854
2384
|
while (tokensGenerated < opts.maxTokens) {
|
|
1855
|
-
if (options.signal?.aborted)
|
|
2385
|
+
if (options.signal?.aborted) {
|
|
2386
|
+
this._recordStopReason('aborted');
|
|
2387
|
+
break;
|
|
2388
|
+
}
|
|
1856
2389
|
if (this._hasFinitenessFallbackWindow() && useBatchPath) {
|
|
1857
2390
|
useBatchPath = false;
|
|
1858
2391
|
}
|
|
@@ -1864,21 +2397,58 @@ export class PipelineGenerator {
|
|
|
1864
2397
|
hasHotVocabularyBatchDecode: hotVocabularyBatchDecodeAvailable,
|
|
1865
2398
|
hasGpuSplitPerLayerInputs,
|
|
1866
2399
|
hasLinearAttentionLayers: hasLinearLayers,
|
|
2400
|
+
modelId: this.#state.modelConfig.modelId ?? executionPlan.kernelPathId,
|
|
2401
|
+
activationDtype: executionPlan.activationDtype,
|
|
2402
|
+
currentSeqLen: this.#state.currentSeqLen,
|
|
2403
|
+
maxDecodeTokens: opts.maxTokens,
|
|
2404
|
+
numLayers: this.#state.modelConfig.numLayers,
|
|
2405
|
+
hiddenSize: this.#state.modelConfig.hiddenSize,
|
|
1867
2406
|
});
|
|
1868
2407
|
const requestedBatchTokens = executionPlan.batchSize * intervalBatches;
|
|
1869
2408
|
const boundedBatchTokens = maxBatchDecodeTokens == null
|
|
1870
2409
|
? requestedBatchTokens
|
|
1871
2410
|
: Math.min(requestedBatchTokens, maxBatchDecodeTokens);
|
|
1872
2411
|
const thisBatchSize = Math.min(boundedBatchTokens, remaining);
|
|
2412
|
+
this.#state.batchingStats.requestedBatchTokens = Math.max(
|
|
2413
|
+
this.#state.batchingStats.requestedBatchTokens ?? 0,
|
|
2414
|
+
requestedBatchTokens
|
|
2415
|
+
);
|
|
2416
|
+
this.#state.batchingStats.effectiveBatchTokens = Math.max(
|
|
2417
|
+
this.#state.batchingStats.effectiveBatchTokens ?? 0,
|
|
2418
|
+
thisBatchSize
|
|
2419
|
+
);
|
|
2420
|
+
if (maxBatchDecodeTokens != null) {
|
|
2421
|
+
this.#state.batchingStats.maxBatchTokenCap = Math.max(
|
|
2422
|
+
this.#state.batchingStats.maxBatchTokenCap ?? 0,
|
|
2423
|
+
maxBatchDecodeTokens
|
|
2424
|
+
);
|
|
2425
|
+
}
|
|
2426
|
+
if (thisBatchSize < requestedBatchTokens) {
|
|
2427
|
+
this.#state.batchingStats.batchClampCount = (this.#state.batchingStats.batchClampCount ?? 0) + 1;
|
|
2428
|
+
}
|
|
1873
2429
|
const lastToken = generatedIds[generatedIds.length - 1];
|
|
2430
|
+
const boundedExecutionPlan = thisBatchSize < requestedBatchTokens
|
|
2431
|
+
? {
|
|
2432
|
+
...executionPlan,
|
|
2433
|
+
batchSize: Math.min(executionPlan.batchSize, thisBatchSize),
|
|
2434
|
+
readbackInterval: readbackInterval == null
|
|
2435
|
+
? null
|
|
2436
|
+
: Math.min(intervalBatches, thisBatchSize),
|
|
2437
|
+
}
|
|
2438
|
+
: executionPlan;
|
|
2439
|
+
const batchOpts = boundedExecutionPlan === executionPlan
|
|
2440
|
+
? opts
|
|
2441
|
+
: { ...opts, executionPlan: boundedExecutionPlan };
|
|
1874
2442
|
|
|
1875
2443
|
try {
|
|
1876
|
-
const batchResult = await this._generateNTokensGPU(lastToken, thisBatchSize, generatedIds,
|
|
2444
|
+
const batchResult = await this._generateNTokensGPU(lastToken, thisBatchSize, generatedIds, batchOpts);
|
|
1877
2445
|
let batchTokens = [];
|
|
1878
2446
|
let hitStop = false;
|
|
2447
|
+
let stopTokenId = null;
|
|
1879
2448
|
for (const tokenId of batchResult.tokens) {
|
|
1880
2449
|
if (isStopToken(tokenId, stopTokenIds, eosToken)) {
|
|
1881
2450
|
hitStop = true;
|
|
2451
|
+
stopTokenId = tokenId;
|
|
1882
2452
|
break;
|
|
1883
2453
|
}
|
|
1884
2454
|
generatedIds.push(tokenId);
|
|
@@ -1901,9 +2471,15 @@ export class PipelineGenerator {
|
|
|
1901
2471
|
if (batchTokens.length > 0 && options.onBatch) options.onBatch(batchTokens);
|
|
1902
2472
|
if (opts.stopSequences.length > 0) {
|
|
1903
2473
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
1904
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq)))
|
|
2474
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
2475
|
+
this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
|
|
2476
|
+
break;
|
|
2477
|
+
}
|
|
2478
|
+
}
|
|
2479
|
+
if (hitStop) {
|
|
2480
|
+
this._recordStopReason('stop-token', stopTokenId);
|
|
2481
|
+
break;
|
|
1905
2482
|
}
|
|
1906
|
-
if (hitStop) break;
|
|
1907
2483
|
if (shouldDisableBatchDecodeAfterShortBatch({
|
|
1908
2484
|
hitStop,
|
|
1909
2485
|
actualCount: batchResult.actualCount,
|
|
@@ -1941,7 +2517,10 @@ export class PipelineGenerator {
|
|
|
1941
2517
|
if (options.onToken) options.onToken(nextToken, tokenText);
|
|
1942
2518
|
}
|
|
1943
2519
|
this._consumeFinitenessFallbackToken(opts);
|
|
1944
|
-
if (isStopToken(nextToken, stopTokenIds, eosToken))
|
|
2520
|
+
if (isStopToken(nextToken, stopTokenIds, eosToken)) {
|
|
2521
|
+
this._recordStopReason('stop-token', nextToken);
|
|
2522
|
+
break;
|
|
2523
|
+
}
|
|
1945
2524
|
}
|
|
1946
2525
|
} else if (opts.speculation?.mode === 'self') {
|
|
1947
2526
|
// Self-speculation: decode one base token plus a configurable burst of
|
|
@@ -1979,11 +2558,17 @@ export class PipelineGenerator {
|
|
|
1979
2558
|
}
|
|
1980
2559
|
this._consumeFinitenessFallbackToken(opts);
|
|
1981
2560
|
|
|
1982
|
-
if (isStopToken(baseToken, stopTokenIds, eosToken))
|
|
2561
|
+
if (isStopToken(baseToken, stopTokenIds, eosToken)) {
|
|
2562
|
+
this._recordStopReason('stop-token', baseToken);
|
|
2563
|
+
break;
|
|
2564
|
+
}
|
|
1983
2565
|
if (tokensGenerated >= opts.maxTokens) break;
|
|
1984
2566
|
if (opts.stopSequences.length > 0) {
|
|
1985
2567
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
1986
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq)))
|
|
2568
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
2569
|
+
this._recordStopReason('stop-sequence', baseToken);
|
|
2570
|
+
break;
|
|
2571
|
+
}
|
|
1987
2572
|
}
|
|
1988
2573
|
|
|
1989
2574
|
for (let specIndex = 0; specIndex < speculativeBurstTokens; specIndex += 1) {
|
|
@@ -2022,19 +2607,27 @@ export class PipelineGenerator {
|
|
|
2022
2607
|
}
|
|
2023
2608
|
|
|
2024
2609
|
if (isStopToken(specToken, stopTokenIds, eosToken)) {
|
|
2610
|
+
this._recordStopReason('stop-token', specToken);
|
|
2025
2611
|
break;
|
|
2026
2612
|
}
|
|
2027
2613
|
if (opts.stopSequences.length > 0) {
|
|
2028
2614
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
2029
2615
|
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
2616
|
+
this._recordStopReason('stop-sequence', specToken);
|
|
2030
2617
|
break;
|
|
2031
2618
|
}
|
|
2032
2619
|
}
|
|
2033
2620
|
}
|
|
2034
|
-
if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken))
|
|
2621
|
+
if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken)) {
|
|
2622
|
+
this._recordStopReason('stop-token', generatedIds[generatedIds.length - 1]);
|
|
2623
|
+
break;
|
|
2624
|
+
}
|
|
2035
2625
|
if (opts.stopSequences.length > 0) {
|
|
2036
2626
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
2037
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq)))
|
|
2627
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
2628
|
+
this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
|
|
2629
|
+
break;
|
|
2630
|
+
}
|
|
2038
2631
|
}
|
|
2039
2632
|
} else {
|
|
2040
2633
|
const tokenStart = performance.now();
|
|
@@ -2088,17 +2681,27 @@ export class PipelineGenerator {
|
|
|
2088
2681
|
log.debug('Decode', `#${tokensGenerated} "${tokenText}" ${tokenTime.toFixed(0)}ms (${tokPerSec.toFixed(2)} tok/s avg)`);
|
|
2089
2682
|
}
|
|
2090
2683
|
|
|
2091
|
-
if (isStopToken(nextToken, stopTokenIds, eosToken))
|
|
2684
|
+
if (isStopToken(nextToken, stopTokenIds, eosToken)) {
|
|
2685
|
+
this._recordStopReason('stop-token', nextToken);
|
|
2686
|
+
break;
|
|
2687
|
+
}
|
|
2092
2688
|
if (opts.stopSequences.length > 0) {
|
|
2093
2689
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
2094
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq)))
|
|
2690
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
2691
|
+
this._recordStopReason('stop-sequence', nextToken);
|
|
2692
|
+
break;
|
|
2693
|
+
}
|
|
2095
2694
|
}
|
|
2096
2695
|
}
|
|
2097
2696
|
}
|
|
2098
2697
|
|
|
2698
|
+
if (!this.#state.stats.stopReason) {
|
|
2699
|
+
this._recordStopReason(tokensGenerated >= opts.maxTokens ? 'max-tokens' : 'completed');
|
|
2700
|
+
}
|
|
2099
2701
|
this.#state.stats.decodeTimeMs = performance.now() - decodeStart;
|
|
2100
2702
|
this.#state.stats.tokensGenerated = tokensGenerated;
|
|
2101
2703
|
this.#state.stats.decodeTokens = tokensGenerated;
|
|
2704
|
+
this.#state.stats.batching = { ...this.#state.batchingStats };
|
|
2102
2705
|
}
|
|
2103
2706
|
|
|
2104
2707
|
async _prefillToHidden(inputIds, opts) {
|
|
@@ -2111,6 +2714,8 @@ export class PipelineGenerator {
|
|
|
2111
2714
|
const numTokens = inputIds.length;
|
|
2112
2715
|
const config = this.#state.modelConfig;
|
|
2113
2716
|
const startPos = this.#state.currentSeqLen;
|
|
2717
|
+
const tracePrefillEnabled = isTraceEnabled('perf');
|
|
2718
|
+
const prefillTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
2114
2719
|
const returnHidden = opts?._returnHidden === true;
|
|
2115
2720
|
const embeddingInputIds = resolvePrefillEmbeddingInputIds(
|
|
2116
2721
|
inputIds,
|
|
@@ -2146,13 +2751,14 @@ export class PipelineGenerator {
|
|
|
2146
2751
|
}
|
|
2147
2752
|
|
|
2148
2753
|
const embedBufferRaw = this.#state.weights.get('embed');
|
|
2149
|
-
if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
|
|
2754
|
+
if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
|
|
2150
2755
|
throw new Error('Embed buffer not found or not a supported buffer type');
|
|
2151
2756
|
}
|
|
2152
2757
|
const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
|
|
2153
2758
|
const embedDtype = isCpuWeightBuffer(embedBufferRaw)
|
|
2154
2759
|
? embedBufferRaw.dtype
|
|
2155
2760
|
: getWeightDtype(embedBufferRaw);
|
|
2761
|
+
const embedMetadata = getWeightMetadata(embedBufferRaw);
|
|
2156
2762
|
if (opts.debug) {
|
|
2157
2763
|
const embedSize = isGpuBufferInstance(embedBuffer) ? embedBuffer.size : 'N/A';
|
|
2158
2764
|
log.debug('Pipeline', `Embed buffer: type=${embedBuffer?.constructor?.name}, size=${embedSize}, dtype=${embedDtype}`);
|
|
@@ -2167,7 +2773,9 @@ export class PipelineGenerator {
|
|
|
2167
2773
|
);
|
|
2168
2774
|
const createRecorder = (label) => {
|
|
2169
2775
|
if (!device || disableCommandBatching) return undefined;
|
|
2170
|
-
return opts.profile
|
|
2776
|
+
return opts.profile
|
|
2777
|
+
? createProfilingRecorder(label, device)
|
|
2778
|
+
: createCommandRecorder(label, { recordLabels: opts.debug === true }, device);
|
|
2171
2779
|
};
|
|
2172
2780
|
const recorder = createRecorder('prefill');
|
|
2173
2781
|
const debugCheckBuffer = this.#state.debug
|
|
@@ -2183,6 +2791,7 @@ export class PipelineGenerator {
|
|
|
2183
2791
|
opts.executionPlan
|
|
2184
2792
|
);
|
|
2185
2793
|
context.currentTokenIds = inputIds;
|
|
2794
|
+
context.diffusionGemmaDecoder = opts?._diffusionGemmaDecoder === true;
|
|
2186
2795
|
context.multimodalBidirectionalSpan = multimodalBidirectionalSpan == null
|
|
2187
2796
|
? null
|
|
2188
2797
|
: {
|
|
@@ -2232,6 +2841,16 @@ export class PipelineGenerator {
|
|
|
2232
2841
|
|
|
2233
2842
|
const activationDtype = opts.executionPlan?.activationDtype ?? this._getEffectiveActivationDtype();
|
|
2234
2843
|
const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
|
|
2844
|
+
if (tracePrefillEnabled) {
|
|
2845
|
+
trace.perf('Prefill phase start', {
|
|
2846
|
+
numTokens,
|
|
2847
|
+
startPos,
|
|
2848
|
+
numLayers: config.numLayers,
|
|
2849
|
+
activationDtype,
|
|
2850
|
+
returnHidden,
|
|
2851
|
+
});
|
|
2852
|
+
}
|
|
2853
|
+
const embedTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
2235
2854
|
let baseEmbeddings = await embed(embeddingInputIds, embedBuffer, {
|
|
2236
2855
|
hiddenSize: config.hiddenSize,
|
|
2237
2856
|
vocabSize: config.vocabSize,
|
|
@@ -2242,18 +2861,29 @@ export class PipelineGenerator {
|
|
|
2242
2861
|
debugProbes: this.#state.runtimeConfig.shared.debug.probes,
|
|
2243
2862
|
operatorDiagnostics: this.#state.operatorDiagnostics,
|
|
2244
2863
|
activationDtype,
|
|
2245
|
-
embeddingDtype: selectRuleValue('inference', 'dtype', '
|
|
2864
|
+
embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
|
|
2865
|
+
embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
|
|
2246
2866
|
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
2247
2867
|
});
|
|
2868
|
+
if (tracePrefillEnabled) {
|
|
2869
|
+
trace.perf('Prefill embed complete', {
|
|
2870
|
+
numTokens,
|
|
2871
|
+
elapsedMs: performance.now() - embedTraceStart,
|
|
2872
|
+
});
|
|
2873
|
+
}
|
|
2248
2874
|
let hiddenStates = baseEmbeddings;
|
|
2249
2875
|
let perLayerInputs = null;
|
|
2876
|
+
const perLayerInputsTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
2250
2877
|
try {
|
|
2251
2878
|
hiddenStates = await applyPrefixEmbeddingOverride(
|
|
2252
2879
|
baseEmbeddings,
|
|
2253
2880
|
embeddingOverride,
|
|
2254
2881
|
config.hiddenSize,
|
|
2255
2882
|
'_prefill',
|
|
2256
|
-
|
|
2883
|
+
{
|
|
2884
|
+
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
2885
|
+
transitionDeclaredBy: resolvePrefixEmbeddingOverrideTransitionDeclaredBy(this.#state.executionV1State),
|
|
2886
|
+
}
|
|
2257
2887
|
);
|
|
2258
2888
|
perLayerInputs = await preparePerLayerInputs(
|
|
2259
2889
|
embeddingInputIds,
|
|
@@ -2264,6 +2894,13 @@ export class PipelineGenerator {
|
|
|
2264
2894
|
pleCache: this.#state.pleCache ?? null,
|
|
2265
2895
|
}
|
|
2266
2896
|
);
|
|
2897
|
+
if (tracePrefillEnabled) {
|
|
2898
|
+
trace.perf('Prefill per-layer inputs complete', {
|
|
2899
|
+
numTokens,
|
|
2900
|
+
elapsedMs: performance.now() - perLayerInputsTraceStart,
|
|
2901
|
+
materialized: Array.isArray(perLayerInputs),
|
|
2902
|
+
});
|
|
2903
|
+
}
|
|
2267
2904
|
} catch (error) {
|
|
2268
2905
|
if (isGpuBufferInstance(hiddenStates?.buffer)) {
|
|
2269
2906
|
releaseBuffer(hiddenStates.buffer);
|
|
@@ -2312,7 +2949,7 @@ export class PipelineGenerator {
|
|
|
2312
2949
|
if (this.#state.finitenessBuffer) {
|
|
2313
2950
|
const device = getDevice();
|
|
2314
2951
|
if (device) {
|
|
2315
|
-
device.queue.writeBuffer(this.#state.finitenessBuffer, 0,
|
|
2952
|
+
device.queue.writeBuffer(this.#state.finitenessBuffer, 0, FINITENESS_RESET_WORDS);
|
|
2316
2953
|
}
|
|
2317
2954
|
}
|
|
2318
2955
|
|
|
@@ -2321,16 +2958,37 @@ export class PipelineGenerator {
|
|
|
2321
2958
|
// Chunked recorder submission: submit every N layers to release tracked intermediate
|
|
2322
2959
|
// buffers, preventing unbounded memory growth during large prefills. Critical for
|
|
2323
2960
|
// replay_prefill models where each decode step re-runs a prefill-style layer pass.
|
|
2324
|
-
const
|
|
2961
|
+
const prefillRecorderChunkLayers = resolvePrefillRecorderChunkLayers({
|
|
2962
|
+
hasGpuSplitPerLayerInputs: context.perLayerInputsSession?.materialization === 'gpu_split_tables',
|
|
2963
|
+
numTokens,
|
|
2964
|
+
});
|
|
2325
2965
|
|
|
2326
2966
|
let currentHiddenBuffer = hiddenStates.buffer;
|
|
2967
|
+
let prefillRecordMs = 0;
|
|
2968
|
+
let prefillSubmitWaitMs = 0;
|
|
2969
|
+
const layerLoopTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
2970
|
+
const traceLayerHealthEnabled = isTraceEnabled('logits');
|
|
2327
2971
|
try {
|
|
2328
2972
|
for (let l = 0; l < config.numLayers; l++) {
|
|
2973
|
+
// Per-layer hard cancellation: when the caller's AbortSignal aborts,
|
|
2974
|
+
// exit the prefill loop between layer dispatches rather than
|
|
2975
|
+
// continuing to burn GPU on superseded work. Granularity is one
|
|
2976
|
+
// layer (~5-50ms), which is the fastest cancel granularity WebGPU
|
|
2977
|
+
// exposes today.
|
|
2978
|
+
if (opts?.signal?.aborted) {
|
|
2979
|
+
const reason = typeof opts.signal.reason === "string" ? opts.signal.reason : "Doppler: prefill aborted";
|
|
2980
|
+
const err = new Error(reason);
|
|
2981
|
+
err.name = "AbortError";
|
|
2982
|
+
err.code = "ABORT_ERR";
|
|
2983
|
+
throw err;
|
|
2984
|
+
}
|
|
2329
2985
|
context.recorder = currentRecorder;
|
|
2330
2986
|
context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
|
|
2331
2987
|
|
|
2332
2988
|
const prevBuffer = currentHiddenBuffer;
|
|
2989
|
+
const layerRecordStart = performance.now();
|
|
2333
2990
|
const layerOutput = await processLayer(l, currentHiddenBuffer, numTokens, true, context);
|
|
2991
|
+
prefillRecordMs += performance.now() - layerRecordStart;
|
|
2334
2992
|
if (!isGpuBufferInstance(layerOutput)) throw new Error('Expected GPUBuffer from processLayer');
|
|
2335
2993
|
currentHiddenBuffer = layerOutput;
|
|
2336
2994
|
releasePerLayerInputBuffer(
|
|
@@ -2346,9 +3004,10 @@ export class PipelineGenerator {
|
|
|
2346
3004
|
|
|
2347
3005
|
const isCheckpoint = useCheckpoints && opts.debugLayers?.includes(l);
|
|
2348
3006
|
const isChunkBoundary = !isCheckpoint
|
|
3007
|
+
&& !traceLayerHealthEnabled
|
|
2349
3008
|
&& currentRecorder
|
|
2350
3009
|
&& l < config.numLayers - 1
|
|
2351
|
-
&& (l + 1) %
|
|
3010
|
+
&& (l + 1) % prefillRecorderChunkLayers === 0;
|
|
2352
3011
|
|
|
2353
3012
|
if (isCheckpoint && currentRecorder) {
|
|
2354
3013
|
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
@@ -2361,6 +3020,27 @@ export class PipelineGenerator {
|
|
|
2361
3020
|
currentRecorder = undefined;
|
|
2362
3021
|
}
|
|
2363
3022
|
|
|
3023
|
+
if (traceLayerHealthEnabled && currentRecorder) {
|
|
3024
|
+
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3025
|
+
currentHiddenBuffer,
|
|
3026
|
+
currentRecorder,
|
|
3027
|
+
'prefill_trace_layer_health_carry'
|
|
3028
|
+
);
|
|
3029
|
+
const traceSubmitStart = performance.now();
|
|
3030
|
+
await currentRecorder.submitAndWait();
|
|
3031
|
+
await recordProfile(currentRecorder);
|
|
3032
|
+
prefillSubmitWaitMs += performance.now() - traceSubmitStart;
|
|
3033
|
+
await traceActivationHealth(
|
|
3034
|
+
`PREFILL_LAYER_${l}_HEALTH`,
|
|
3035
|
+
currentHiddenBuffer,
|
|
3036
|
+
activationDtype,
|
|
3037
|
+
numTokens * config.hiddenSize
|
|
3038
|
+
);
|
|
3039
|
+
currentRecorder = l < config.numLayers - 1
|
|
3040
|
+
? createRecorder('prefill-trace')
|
|
3041
|
+
: undefined;
|
|
3042
|
+
}
|
|
3043
|
+
|
|
2364
3044
|
const shouldDebug = opts.debug && currentHiddenBuffer && (!recorder || isCheckpoint);
|
|
2365
3045
|
if (shouldDebug && !currentRecorder) {
|
|
2366
3046
|
const device = getDevice();
|
|
@@ -2408,11 +3088,50 @@ export class PipelineGenerator {
|
|
|
2408
3088
|
currentRecorder,
|
|
2409
3089
|
'prefill_chunk_carry'
|
|
2410
3090
|
);
|
|
2411
|
-
|
|
2412
|
-
|
|
3091
|
+
// Chunk boundary exists only to bound intermediate buffer lifetime.
|
|
3092
|
+
// When the runtime opts into async chunk submit AND profile timings
|
|
3093
|
+
// are not being collected, skip the CPU-GPU wait: queue order is
|
|
3094
|
+
// preserved across submits and deferred cleanup still releases
|
|
3095
|
+
// tracked buffers when GPU work completes. Profile runs keep the
|
|
3096
|
+
// sync path because resolveProfileTimings requires the submitted
|
|
3097
|
+
// work to be done.
|
|
3098
|
+
const chunkSubmitMode = resolvePrefillChunkSubmitMode(
|
|
3099
|
+
this.#state.runtimeConfig,
|
|
3100
|
+
this.#state.modelConfig
|
|
3101
|
+
);
|
|
3102
|
+
const chunkSubmitStart = performance.now();
|
|
3103
|
+
if (chunkSubmitMode === 'async' && !opts.profile) {
|
|
3104
|
+
currentRecorder.submit({ cleanup: 'deferred' });
|
|
3105
|
+
} else {
|
|
3106
|
+
await currentRecorder.submitAndWait();
|
|
3107
|
+
await recordProfile(currentRecorder);
|
|
3108
|
+
}
|
|
3109
|
+
prefillSubmitWaitMs += performance.now() - chunkSubmitStart;
|
|
3110
|
+
await traceActivationHealth(
|
|
3111
|
+
`PREFILL_LAYER_${l}_HEALTH`,
|
|
3112
|
+
currentHiddenBuffer,
|
|
3113
|
+
activationDtype,
|
|
3114
|
+
numTokens * config.hiddenSize
|
|
3115
|
+
);
|
|
3116
|
+
if (tracePrefillEnabled) {
|
|
3117
|
+
trace.perf('Prefill chunk submitted', {
|
|
3118
|
+
layer: l,
|
|
3119
|
+
elapsedMs: performance.now() - layerLoopTraceStart,
|
|
3120
|
+
prefillRecordMs,
|
|
3121
|
+
prefillSubmitWaitMs,
|
|
3122
|
+
});
|
|
3123
|
+
}
|
|
2413
3124
|
currentRecorder = createRecorder('prefill-chunk');
|
|
2414
3125
|
}
|
|
2415
3126
|
}
|
|
3127
|
+
if (tracePrefillEnabled) {
|
|
3128
|
+
trace.perf('Prefill layer loop recorded', {
|
|
3129
|
+
numTokens,
|
|
3130
|
+
elapsedMs: performance.now() - layerLoopTraceStart,
|
|
3131
|
+
prefillRecordMs,
|
|
3132
|
+
prefillSubmitWaitMs,
|
|
3133
|
+
});
|
|
3134
|
+
}
|
|
2416
3135
|
} finally {
|
|
2417
3136
|
context.perLayerInputBuffer = null;
|
|
2418
3137
|
if (perLayerInputs) {
|
|
@@ -2426,8 +3145,26 @@ export class PipelineGenerator {
|
|
|
2426
3145
|
}
|
|
2427
3146
|
}
|
|
2428
3147
|
releaseSharedAttentionState(context.sharedAttentionState, currentRecorder);
|
|
3148
|
+
this.#state.stats.prefillRecordMs = (this.#state.stats.prefillRecordMs ?? 0) + prefillRecordMs;
|
|
3149
|
+
this.#state.stats.prefillSubmitWaitMs = (this.#state.stats.prefillSubmitWaitMs ?? 0) + prefillSubmitWaitMs;
|
|
2429
3150
|
}
|
|
2430
3151
|
|
|
3152
|
+
{
|
|
3153
|
+
const tsirFixtureCfg = this.#state.operatorDiagnostics?.tsirFixture ?? null;
|
|
3154
|
+
if (tsirFixtureCfg && Array.isArray(tsirFixtureCfg.pendingReads) && tsirFixtureCfg.pendingReads.length > 0) {
|
|
3155
|
+
if (currentRecorder) {
|
|
3156
|
+
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3157
|
+
currentHiddenBuffer,
|
|
3158
|
+
currentRecorder,
|
|
3159
|
+
'prefill_tsir_drain_carry'
|
|
3160
|
+
);
|
|
3161
|
+
await currentRecorder.submitAndWait();
|
|
3162
|
+
await recordProfile(currentRecorder);
|
|
3163
|
+
currentRecorder = undefined;
|
|
3164
|
+
}
|
|
3165
|
+
await drainPendingTsirReads(tsirFixtureCfg);
|
|
3166
|
+
}
|
|
3167
|
+
}
|
|
2431
3168
|
if (this.#state.finitenessBuffer) {
|
|
2432
3169
|
if (currentRecorder) {
|
|
2433
3170
|
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
@@ -2506,6 +3243,7 @@ export class PipelineGenerator {
|
|
|
2506
3243
|
&& !this.#state.disableRecordedLogits
|
|
2507
3244
|
&& numTokens === 1;
|
|
2508
3245
|
if (currentRecorder && canRecordLogits) {
|
|
3246
|
+
const logitsTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
2509
3247
|
const recorded = await recordLogitsGPU(
|
|
2510
3248
|
currentRecorder,
|
|
2511
3249
|
currentHiddenBuffer,
|
|
@@ -2526,6 +3264,13 @@ export class PipelineGenerator {
|
|
|
2526
3264
|
const logitsData = await readBufferSlice(recorded.logitsBuffer, lastLogitsOffset, lastLogitsSize);
|
|
2527
3265
|
releaseBuffer(recorded.logitsBuffer);
|
|
2528
3266
|
lastLogits = decodeReadback(logitsData, recorded.logitsDtype);
|
|
3267
|
+
if (tracePrefillEnabled) {
|
|
3268
|
+
trace.perf('Prefill recorded logits complete', {
|
|
3269
|
+
numTokens,
|
|
3270
|
+
vocabSize: logitsVocabSize,
|
|
3271
|
+
elapsedMs: performance.now() - logitsTraceStart,
|
|
3272
|
+
});
|
|
3273
|
+
}
|
|
2529
3274
|
|
|
2530
3275
|
const health = getLogitsHealth(lastLogits);
|
|
2531
3276
|
if (health.nanCount > 0 || health.infCount > 0 || health.nonZeroCount === 0) {
|
|
@@ -2564,6 +3309,7 @@ export class PipelineGenerator {
|
|
|
2564
3309
|
|
|
2565
3310
|
releaseBuffer(currentHiddenBuffer);
|
|
2566
3311
|
} else {
|
|
3312
|
+
const logitsTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
2567
3313
|
if (currentRecorder) {
|
|
2568
3314
|
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
2569
3315
|
currentHiddenBuffer,
|
|
@@ -2591,6 +3337,13 @@ export class PipelineGenerator {
|
|
|
2591
3337
|
? logits
|
|
2592
3338
|
: extractLastPositionLogits(logits, numTokens, logitsVocabSize);
|
|
2593
3339
|
releaseBuffer(currentHiddenBuffer);
|
|
3340
|
+
if (tracePrefillEnabled) {
|
|
3341
|
+
trace.perf('Prefill logits complete', {
|
|
3342
|
+
numTokens,
|
|
3343
|
+
vocabSize: logitsVocabSize,
|
|
3344
|
+
elapsedMs: performance.now() - logitsTraceStart,
|
|
3345
|
+
});
|
|
3346
|
+
}
|
|
2594
3347
|
}
|
|
2595
3348
|
|
|
2596
3349
|
this.#state.currentSeqLen = startPos + numTokens;
|
|
@@ -2610,6 +3363,9 @@ export class PipelineGenerator {
|
|
|
2610
3363
|
if (opts.debug) {
|
|
2611
3364
|
logitsSanity(lastLogits, 'Prefill', (tokens) => resolveTokenText(this.#state.tokenizer, tokens));
|
|
2612
3365
|
}
|
|
3366
|
+
if (isTraceEnabled('logits')) {
|
|
3367
|
+
trace.logits('PREFILL_LOGITS_HEALTH', getLogitsHealth(lastLogits));
|
|
3368
|
+
}
|
|
2613
3369
|
|
|
2614
3370
|
if (opts.debug) {
|
|
2615
3371
|
if (this.#state.kvCache?.hasGPUCache?.()) {
|
|
@@ -2619,6 +3375,13 @@ export class PipelineGenerator {
|
|
|
2619
3375
|
}
|
|
2620
3376
|
}
|
|
2621
3377
|
|
|
3378
|
+
if (tracePrefillEnabled) {
|
|
3379
|
+
trace.perf('Prefill phase complete', {
|
|
3380
|
+
numTokens,
|
|
3381
|
+
totalMs: performance.now() - prefillTraceStart,
|
|
3382
|
+
});
|
|
3383
|
+
}
|
|
3384
|
+
|
|
2622
3385
|
return lastLogits;
|
|
2623
3386
|
}
|
|
2624
3387
|
|