doppler-gpu 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +67 -34
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.js +5 -1
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/version.js +1 -1
|
@@ -7,7 +7,18 @@ import { markWarmed as markKernelCacheWarmed } from '../../../gpu/kernel-selecti
|
|
|
7
7
|
import { resetSubmitStats, logSubmitStats } from '../../../gpu/submit-tracker.js';
|
|
8
8
|
import { createCommandRecorder, createProfilingRecorder, CommandRecorder } from '../../../gpu/command-recorder.js';
|
|
9
9
|
import { allowReadback } from '../../../gpu/perf-guards.js';
|
|
10
|
-
import { log, trace } from '../../../debug/index.js';
|
|
10
|
+
import { log, trace, isTraceEnabled } from '../../../debug/index.js';
|
|
11
|
+
import {
|
|
12
|
+
runMatmul,
|
|
13
|
+
runRMSNorm,
|
|
14
|
+
runGeLU,
|
|
15
|
+
runResidualAdd,
|
|
16
|
+
runScale,
|
|
17
|
+
runSoftmax,
|
|
18
|
+
runSoftEmbeddingSplitF16,
|
|
19
|
+
runSoftEmbeddingLogitsF16,
|
|
20
|
+
} from '../../../gpu/kernel-selector.js';
|
|
21
|
+
import { runDiffusionGemmaCanvasStats } from '../../../gpu/kernels/diffusion-gemma-sampling.js';
|
|
11
22
|
import {
|
|
12
23
|
CAPTURE_LEVELS,
|
|
13
24
|
createDefaultCaptureConfig,
|
|
@@ -21,9 +32,9 @@ import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './samplin
|
|
|
21
32
|
import { createKVCache, isStopToken } from './init.js';
|
|
22
33
|
import { embed } from './embed.js';
|
|
23
34
|
import { processLayer } from './layer.js';
|
|
24
|
-
import { computeLogits, recordLogitsGPU, extractLastPositionLogits, applySoftcapping } from './logits/index.js';
|
|
35
|
+
import { computeLogits, computeLogitsGPU, recordLogitsGPU, extractLastPositionLogits, applySoftcapping } from './logits/index.js';
|
|
25
36
|
import { OperatorEventEmitter } from './operator-events.js';
|
|
26
|
-
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, getWeightDtype } from '../../../gpu/weight-buffer.js';
|
|
37
|
+
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype, getWeightMetadata, getLayout } from '../../../gpu/weight-buffer.js';
|
|
27
38
|
import {
|
|
28
39
|
decodeStep,
|
|
29
40
|
decodeStepLogits,
|
|
@@ -84,6 +95,11 @@ import {
|
|
|
84
95
|
hasRangeBackedPerLayerInputEmbeddings,
|
|
85
96
|
} from './per-layer-inputs.js';
|
|
86
97
|
import { createTensor } from '../../../gpu/tensor.js';
|
|
98
|
+
import { getQKNormOnesBuffer } from './attention/types.js';
|
|
99
|
+
import {
|
|
100
|
+
getWeightBuffer as getPipelineWeightBuffer,
|
|
101
|
+
getNormWeightBuffer as getPipelineNormWeightBuffer,
|
|
102
|
+
} from './weights.js';
|
|
87
103
|
|
|
88
104
|
// Extracted standalone helpers
|
|
89
105
|
import {
|
|
@@ -93,7 +109,9 @@ import {
|
|
|
93
109
|
resolvePrefillEmbeddingInputIds,
|
|
94
110
|
resolvePrefillMultimodalBidirectionalSpan,
|
|
95
111
|
applyPrefixEmbeddingOverride,
|
|
112
|
+
resolvePrefixEmbeddingOverrideTransitionDeclaredBy,
|
|
96
113
|
shouldDisablePrefillCommandBatching,
|
|
114
|
+
resolveEffectivePrefillTokenChunkSize,
|
|
97
115
|
} from './generator-prefill-helpers.js';
|
|
98
116
|
import {
|
|
99
117
|
shouldDisableBatchDecodeAfterShortBatch,
|
|
@@ -111,6 +129,260 @@ import {
|
|
|
111
129
|
cloneRuntimeInferenceWithKVDtype,
|
|
112
130
|
} from './generator-decode-policy.js';
|
|
113
131
|
|
|
132
|
+
const SPECIAL_LIKE_TOKEN_RE = /^(<pad>|<unused\d*>|<eos>|<bos>|<s>|<\/s>|\[PAD\]|\[UNK\]|\[SEP\]|\[CLS\]|<[^>\n]{1,32}>)$/i;
|
|
133
|
+
const FINITENESS_RESET_WORDS = new Uint32Array(4);
|
|
134
|
+
const tokenizerSuppressionCache = new WeakMap();
|
|
135
|
+
const PREFILL_CHUNK_SUBMIT_MODES = new Set(['sync', 'async']);
|
|
136
|
+
|
|
137
|
+
export function resolvePrefillChunkSubmitMode(runtimeConfig, modelConfig) {
|
|
138
|
+
const runtimeSubmit = runtimeConfig?.inference?.session?.prefillChunkSubmitMode;
|
|
139
|
+
const manifestSubmit = modelConfig?.sessionSettings?.prefillChunkSubmitMode;
|
|
140
|
+
const submit = (runtimeSubmit !== undefined && runtimeSubmit !== null)
|
|
141
|
+
? runtimeSubmit
|
|
142
|
+
: manifestSubmit;
|
|
143
|
+
if (submit === undefined || submit === null) {
|
|
144
|
+
throw new Error('[Pipeline] runtime.inference.session.prefillChunkSubmitMode is required.');
|
|
145
|
+
}
|
|
146
|
+
if (!PREFILL_CHUNK_SUBMIT_MODES.has(submit)) {
|
|
147
|
+
throw new Error(
|
|
148
|
+
`[Pipeline] runtime.inference.session.prefillChunkSubmitMode must be "sync" or "async"; got "${String(submit)}".`
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
return submit;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function getTokenizerSuppressionCache(tokenizer) {
|
|
155
|
+
let cache = tokenizerSuppressionCache.get(tokenizer);
|
|
156
|
+
if (!cache) {
|
|
157
|
+
cache = new Map();
|
|
158
|
+
tokenizerSuppressionCache.set(tokenizer, cache);
|
|
159
|
+
}
|
|
160
|
+
return cache;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function decodeSingleTokenForSuppression(tokenizer, tokenId) {
|
|
164
|
+
if (!tokenizer || typeof tokenizer.decode !== 'function') {
|
|
165
|
+
return '';
|
|
166
|
+
}
|
|
167
|
+
try {
|
|
168
|
+
return tokenizer.decode([tokenId], false, false);
|
|
169
|
+
} catch {
|
|
170
|
+
return '';
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function collectTokenizerSuppressedTokenIds(tokenizer, vocabSize, samplingConfig) {
|
|
175
|
+
if (!tokenizer || !Number.isInteger(vocabSize) || vocabSize < 1) {
|
|
176
|
+
return [];
|
|
177
|
+
}
|
|
178
|
+
if (samplingConfig.suppressSpecialTokens !== true && samplingConfig.suppressSpecialLikeTokens !== true) {
|
|
179
|
+
return [];
|
|
180
|
+
}
|
|
181
|
+
const cache = getTokenizerSuppressionCache(tokenizer);
|
|
182
|
+
const cacheKey = [
|
|
183
|
+
vocabSize,
|
|
184
|
+
samplingConfig.suppressSpecialTokens === true ? 'special' : 'plain',
|
|
185
|
+
samplingConfig.suppressSpecialLikeTokens === true ? 'specialLike' : 'literal',
|
|
186
|
+
].join(':');
|
|
187
|
+
const cached = cache.get(cacheKey);
|
|
188
|
+
if (cached) {
|
|
189
|
+
return cached;
|
|
190
|
+
}
|
|
191
|
+
const tokenIds = [];
|
|
192
|
+
for (let tokenId = 0; tokenId < vocabSize; tokenId++) {
|
|
193
|
+
const tokenizerSpecial = samplingConfig.suppressSpecialTokens === true
|
|
194
|
+
&& typeof tokenizer.isSpecialToken === 'function'
|
|
195
|
+
&& tokenizer.isSpecialToken(tokenId);
|
|
196
|
+
const specialLike = samplingConfig.suppressSpecialLikeTokens === true
|
|
197
|
+
&& SPECIAL_LIKE_TOKEN_RE.test(decodeSingleTokenForSuppression(tokenizer, tokenId).trim());
|
|
198
|
+
if (tokenizerSpecial || specialLike) {
|
|
199
|
+
tokenIds.push(tokenId);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
cache.set(cacheKey, tokenIds);
|
|
203
|
+
return tokenIds;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
function resolveSuppressedSamplingTokenIds(state, samplingConfig) {
|
|
207
|
+
const vocabSize = state.modelConfig?.vocabSize;
|
|
208
|
+
const suppressed = new Set(samplingConfig.suppressTokenIds);
|
|
209
|
+
const stopTokenIds = new Set(state.modelConfig?.stopTokenIds ?? []);
|
|
210
|
+
const eosToken = state.tokenizer?.getSpecialTokens?.()?.eos;
|
|
211
|
+
if (Number.isInteger(eosToken)) {
|
|
212
|
+
stopTokenIds.add(eosToken);
|
|
213
|
+
}
|
|
214
|
+
for (const tokenId of collectTokenizerSuppressedTokenIds(state.tokenizer, vocabSize, samplingConfig)) {
|
|
215
|
+
if (!stopTokenIds.has(tokenId)) {
|
|
216
|
+
suppressed.add(tokenId);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
return [...suppressed];
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
async function traceActivationHealth(label, buffer, dtype, elementCount) {
|
|
223
|
+
if (!isTraceEnabled('logits') || !isGpuBufferInstance(buffer)) {
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
|
|
227
|
+
const data = await readBuffer(buffer, elementCount * bytesPerElement);
|
|
228
|
+
trace.logits(label, getLogitsHealth(decodeReadback(data, dtype)));
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function ownsBorrowedWeightBuffer(weight) {
|
|
232
|
+
return !isGpuBufferInstance(weight) && !isWeightBuffer(weight);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
function borrowLinearWeight(weight, label) {
|
|
236
|
+
if (!weight) {
|
|
237
|
+
throw new Error(`DiffusionGemma self-conditioning missing ${label}.`);
|
|
238
|
+
}
|
|
239
|
+
if (isSplitWeightBuffer(weight)) {
|
|
240
|
+
throw new Error(
|
|
241
|
+
`DiffusionGemma self-conditioning does not support split weight storage for ${label}.`
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
return {
|
|
245
|
+
value: getPipelineWeightBuffer(weight, label),
|
|
246
|
+
owned: ownsBorrowedWeightBuffer(weight),
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function borrowNormWeight(weight, label) {
|
|
251
|
+
if (!weight) {
|
|
252
|
+
throw new Error(`DiffusionGemma self-conditioning missing ${label}.`);
|
|
253
|
+
}
|
|
254
|
+
if (isSplitWeightBuffer(weight)) {
|
|
255
|
+
throw new Error(
|
|
256
|
+
`DiffusionGemma self-conditioning does not support split norm storage for ${label}.`
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
return {
|
|
260
|
+
value: getPipelineNormWeightBuffer(weight, label),
|
|
261
|
+
owned: ownsBorrowedWeightBuffer(weight),
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
function releaseBorrowedWeight(borrowed) {
|
|
266
|
+
if (!borrowed?.owned) {
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
const value = borrowed.value;
|
|
270
|
+
releaseBuffer(isWeightBuffer(value) ? value.buffer : value);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function canUseChunkedSoftEmbeddingLogits(logitsState, embeddingWeight, embeddingTranspose) {
|
|
274
|
+
return logitsState != null
|
|
275
|
+
&& isWeightBuffer(embeddingWeight)
|
|
276
|
+
&& getWeightDtype(embeddingWeight) === 'f16'
|
|
277
|
+
&& getLayout(embeddingWeight) === 'row'
|
|
278
|
+
&& embeddingTranspose !== true;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function resolveDiffusionGemmaSoftEmbeddingChunkRows(runtimeConfig) {
|
|
282
|
+
const value = runtimeConfig?.inference?.diffusionGemma?.softEmbeddingLogitsChunkRows;
|
|
283
|
+
if (value == null) {
|
|
284
|
+
return undefined;
|
|
285
|
+
}
|
|
286
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
287
|
+
throw new Error(
|
|
288
|
+
'runtime.inference.diffusionGemma.softEmbeddingLogitsChunkRows must be a positive integer.'
|
|
289
|
+
);
|
|
290
|
+
}
|
|
291
|
+
return value;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
function normalizeCanvasTokenIds(canvas, context) {
|
|
295
|
+
if (!Array.isArray(canvas) && !ArrayBuffer.isView(canvas)) {
|
|
296
|
+
throw new Error(`[DiffusionGemma] ${context}.canvas must be an array or typed array of token IDs.`);
|
|
297
|
+
}
|
|
298
|
+
return Array.from(canvas, (value, index) => {
|
|
299
|
+
if (!Number.isFinite(value) || Math.floor(value) !== value || value < 0) {
|
|
300
|
+
throw new Error(`[DiffusionGemma] ${context}.canvas[${index}] must be a non-negative integer token ID.`);
|
|
301
|
+
}
|
|
302
|
+
return value;
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function normalizeSelfConditioningLogits(logits, canvasLength, vocabSize) {
|
|
307
|
+
if (logits == null) {
|
|
308
|
+
return null;
|
|
309
|
+
}
|
|
310
|
+
const expected = canvasLength * vocabSize;
|
|
311
|
+
const values = logits instanceof Float32Array
|
|
312
|
+
? logits
|
|
313
|
+
: (Array.isArray(logits) ? Float32Array.from(logits) : null);
|
|
314
|
+
if (!values) {
|
|
315
|
+
throw new Error('[DiffusionGemma] selfConditioningLogits must be a Float32Array or number array.');
|
|
316
|
+
}
|
|
317
|
+
if (values.length !== expected) {
|
|
318
|
+
throw new Error(
|
|
319
|
+
`[DiffusionGemma] selfConditioningLogits length mismatch: expected ${expected}, got ${values.length}.`
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
return values;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function normalizeSelfConditioningLogitsState(logits, canvasLength, vocabSize) {
|
|
326
|
+
if (logits == null || !isGpuBufferInstance(logits.logitsBuffer)) {
|
|
327
|
+
return null;
|
|
328
|
+
}
|
|
329
|
+
const dtype = logits.logitsDtype;
|
|
330
|
+
if (dtype !== 'f32') {
|
|
331
|
+
throw new Error(`[DiffusionGemma] GPU selfConditioningLogits require f32 logits, got "${dtype}".`);
|
|
332
|
+
}
|
|
333
|
+
if (logits.vocabSize !== vocabSize) {
|
|
334
|
+
throw new Error(
|
|
335
|
+
`[DiffusionGemma] GPU selfConditioningLogits vocab mismatch: expected ${vocabSize}, got ${logits.vocabSize}.`
|
|
336
|
+
);
|
|
337
|
+
}
|
|
338
|
+
if (logits.canvasLength !== canvasLength) {
|
|
339
|
+
throw new Error(
|
|
340
|
+
`[DiffusionGemma] GPU selfConditioningLogits canvas mismatch: expected ${canvasLength}, got ${logits.canvasLength}.`
|
|
341
|
+
);
|
|
342
|
+
}
|
|
343
|
+
const temperature = logits.temperature ?? 1.0;
|
|
344
|
+
if (typeof temperature !== 'number' || !Number.isFinite(temperature) || temperature <= 0) {
|
|
345
|
+
throw new Error('[DiffusionGemma] GPU selfConditioningLogits temperature must be positive.');
|
|
346
|
+
}
|
|
347
|
+
return {
|
|
348
|
+
logitsBuffer: logits.logitsBuffer,
|
|
349
|
+
logitsDtype: dtype,
|
|
350
|
+
temperature,
|
|
351
|
+
releaseOnUse: logits.releaseOnUse !== false,
|
|
352
|
+
};
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
function normalizeSelfConditioningSoftEmbeddingState(state, canvasLength, hiddenSize) {
|
|
356
|
+
if (state?.kind !== 'soft_embedding') {
|
|
357
|
+
return null;
|
|
358
|
+
}
|
|
359
|
+
if (!isGpuBufferInstance(state.buffer)) {
|
|
360
|
+
throw new Error('[DiffusionGemma] GPU selfConditioning soft embedding requires a GPU buffer.');
|
|
361
|
+
}
|
|
362
|
+
const dtype = state.dtype ?? 'f32';
|
|
363
|
+
if (dtype !== 'f32') {
|
|
364
|
+
throw new Error(`[DiffusionGemma] GPU selfConditioning soft embedding requires f32 dtype, got "${dtype}".`);
|
|
365
|
+
}
|
|
366
|
+
if (state.canvasLength !== canvasLength) {
|
|
367
|
+
throw new Error(
|
|
368
|
+
`[DiffusionGemma] GPU selfConditioning soft embedding canvas mismatch: ` +
|
|
369
|
+
`expected ${canvasLength}, got ${state.canvasLength}.`
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
if (state.hiddenSize !== hiddenSize) {
|
|
373
|
+
throw new Error(
|
|
374
|
+
`[DiffusionGemma] GPU selfConditioning soft embedding hidden mismatch: ` +
|
|
375
|
+
`expected ${hiddenSize}, got ${state.hiddenSize}.`
|
|
376
|
+
);
|
|
377
|
+
}
|
|
378
|
+
return {
|
|
379
|
+
buffer: state.buffer,
|
|
380
|
+
dtype,
|
|
381
|
+
releaseOnUse: state.releaseOnUse !== false,
|
|
382
|
+
scaled: state.scaled === true,
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
|
|
114
386
|
let intentBundleModulePromise = null;
|
|
115
387
|
|
|
116
388
|
async function getExperimentalIntentBundleModule() {
|
|
@@ -256,8 +528,14 @@ export class PipelineGenerator {
|
|
|
256
528
|
this.#state.stats.ttftMs = 0;
|
|
257
529
|
this.#state.stats.decodeTimeMs = 0;
|
|
258
530
|
this.#state.stats.decodeRecordMs = 0;
|
|
531
|
+
this.#state.stats.decodeRecordOps = 0;
|
|
532
|
+
this.#state.stats.decodeRecordPasses = 0;
|
|
533
|
+
this.#state.stats.decodeRecordOpLabels = {};
|
|
259
534
|
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
260
535
|
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
536
|
+
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
537
|
+
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
538
|
+
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
261
539
|
this.#state.decodeStepCount = 0;
|
|
262
540
|
this.#state.disableRecordedLogits = false;
|
|
263
541
|
this.#state.disableFusedDecode = false;
|
|
@@ -267,6 +545,12 @@ export class PipelineGenerator {
|
|
|
267
545
|
totalBatchedTimeMs: 0,
|
|
268
546
|
totalUnbatchedTimeMs: 0,
|
|
269
547
|
gpuSubmissions: 0,
|
|
548
|
+
requestedBatchTokens: 0,
|
|
549
|
+
effectiveBatchTokens: 0,
|
|
550
|
+
executedBatchTokens: 0,
|
|
551
|
+
resolvedBatchTokens: 0,
|
|
552
|
+
maxBatchTokenCap: null,
|
|
553
|
+
batchClampCount: 0,
|
|
270
554
|
};
|
|
271
555
|
resetActiveExecutionPlan(this.#state);
|
|
272
556
|
this.#state.decodeRing?.reset();
|
|
@@ -339,6 +623,7 @@ export class PipelineGenerator {
|
|
|
339
623
|
topK: opts.topK,
|
|
340
624
|
padTokenId,
|
|
341
625
|
seed: opts.seed,
|
|
626
|
+
suppressTokenIds: opts.suppressTokenIds,
|
|
342
627
|
});
|
|
343
628
|
if (typeof opts.onLogits === 'function') {
|
|
344
629
|
opts.onLogits(sampledLogits, {
|
|
@@ -349,6 +634,65 @@ export class PipelineGenerator {
|
|
|
349
634
|
return tokenId;
|
|
350
635
|
}
|
|
351
636
|
|
|
637
|
+
_resolvePrefillTokenChunkSize(inputIds) {
|
|
638
|
+
const chunkSize = resolveEffectivePrefillTokenChunkSize(this.#state);
|
|
639
|
+
if (chunkSize === undefined) {
|
|
640
|
+
throw new Error('inference.session.prefillTokenChunkSize is required; use null to disable token-chunked prefill.');
|
|
641
|
+
}
|
|
642
|
+
if (chunkSize === null) {
|
|
643
|
+
return null;
|
|
644
|
+
}
|
|
645
|
+
if (!Number.isInteger(chunkSize) || chunkSize <= 0) {
|
|
646
|
+
throw new Error('inference.session.prefillTokenChunkSize must be null or a positive integer.');
|
|
647
|
+
}
|
|
648
|
+
return chunkSize < inputIds.length ? chunkSize : null;
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
async _commitPrefillHiddenChunk(prefillResult) {
|
|
652
|
+
const {
|
|
653
|
+
numTokens,
|
|
654
|
+
startPos,
|
|
655
|
+
currentRecorder,
|
|
656
|
+
recordProfile,
|
|
657
|
+
currentHiddenBuffer,
|
|
658
|
+
} = prefillResult;
|
|
659
|
+
|
|
660
|
+
try {
|
|
661
|
+
if (currentRecorder) {
|
|
662
|
+
await currentRecorder.submitAndWait();
|
|
663
|
+
await recordProfile(currentRecorder);
|
|
664
|
+
} else {
|
|
665
|
+
const device = getDevice();
|
|
666
|
+
if (device) {
|
|
667
|
+
await device.queue.onSubmittedWorkDone();
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
this.#state.currentSeqLen = startPos + numTokens;
|
|
671
|
+
} finally {
|
|
672
|
+
releaseBuffer(currentHiddenBuffer);
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
async _prefillInputIdsToLogits(inputIds, opts) {
|
|
677
|
+
const chunkSize = this._resolvePrefillTokenChunkSize(inputIds);
|
|
678
|
+
if (chunkSize === null) {
|
|
679
|
+
return this._prefill(inputIds, opts);
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
for (let offset = 0; offset < inputIds.length; offset += chunkSize) {
|
|
683
|
+
const end = Math.min(offset + chunkSize, inputIds.length);
|
|
684
|
+
const chunk = inputIds.slice(offset, end);
|
|
685
|
+
const isFinalChunk = end === inputIds.length;
|
|
686
|
+
if (isFinalChunk) {
|
|
687
|
+
return this._prefill(chunk, opts);
|
|
688
|
+
}
|
|
689
|
+
const prefillResult = await this._prefillToHidden(chunk, opts);
|
|
690
|
+
await this._commitPrefillHiddenChunk(prefillResult);
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
return this._prefill(inputIds, opts);
|
|
694
|
+
}
|
|
695
|
+
|
|
352
696
|
async _prefillPromptToLogits(prompt, opts, contextLabel) {
|
|
353
697
|
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
354
698
|
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, contextLabel, opts.inputIds);
|
|
@@ -357,8 +701,9 @@ export class PipelineGenerator {
|
|
|
357
701
|
}
|
|
358
702
|
|
|
359
703
|
let logits;
|
|
704
|
+
const runPrefill = () => this._prefillInputIdsToLogits(inputIds, opts);
|
|
360
705
|
try {
|
|
361
|
-
logits = await
|
|
706
|
+
logits = await runPrefill();
|
|
362
707
|
} catch (error) {
|
|
363
708
|
if (!this._shouldUseFinitenessFallback(error, contextLabel)) {
|
|
364
709
|
throw error;
|
|
@@ -368,7 +713,7 @@ export class PipelineGenerator {
|
|
|
368
713
|
opts,
|
|
369
714
|
contextLabel,
|
|
370
715
|
opts.maxTokens ?? 1,
|
|
371
|
-
|
|
716
|
+
runPrefill,
|
|
372
717
|
prefillStartSeqLen
|
|
373
718
|
);
|
|
374
719
|
}
|
|
@@ -423,12 +768,21 @@ export class PipelineGenerator {
|
|
|
423
768
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
424
769
|
this.#state.stats.gpuTimeDecodeMs = undefined;
|
|
425
770
|
this.#state.stats.decodeRecordMs = 0;
|
|
771
|
+
this.#state.stats.decodeRecordOps = 0;
|
|
772
|
+
this.#state.stats.decodeRecordPasses = 0;
|
|
773
|
+
this.#state.stats.decodeRecordOpLabels = {};
|
|
426
774
|
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
427
775
|
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
776
|
+
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
777
|
+
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
778
|
+
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
428
779
|
this.#state.stats.prefillRecordMs = 0;
|
|
429
780
|
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
430
781
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
431
782
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
783
|
+
this.#state.stats.singleTokenReadbackMapWaitMs = 0;
|
|
784
|
+
this.#state.stats.singleTokenReadbackCleanupMs = 0;
|
|
785
|
+
this.#state.stats.singleTokenReadbackCopyMs = 0;
|
|
432
786
|
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
433
787
|
this.#state.stats.pleHotVocabularyHits = 0;
|
|
434
788
|
this.#state.stats.pleHotVocabularyMisses = 0;
|
|
@@ -452,6 +806,7 @@ export class PipelineGenerator {
|
|
|
452
806
|
opts.topP = samplingConfig.topP;
|
|
453
807
|
opts.topK = samplingConfig.topK;
|
|
454
808
|
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
809
|
+
opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this.#state, samplingConfig);
|
|
455
810
|
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
456
811
|
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
457
812
|
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
@@ -824,7 +1179,7 @@ export class PipelineGenerator {
|
|
|
824
1179
|
// Generation Public API
|
|
825
1180
|
// ==========================================================================
|
|
826
1181
|
|
|
827
|
-
|
|
1182
|
+
/*
|
|
828
1183
|
* Truncate the KV cache back to `seqLen` tokens and set `currentSeqLen` to
|
|
829
1184
|
* match. Intended for "prefix-reuse" workflows where a caller wants to run
|
|
830
1185
|
* several decodes that share a common prompt prefix: prefill once with the
|
|
@@ -850,6 +1205,574 @@ export class PipelineGenerator {
|
|
|
850
1205
|
this.#state.currentSeqLen = target;
|
|
851
1206
|
}
|
|
852
1207
|
|
|
1208
|
+
async _createDiffusionGemmaSelfConditioningEmbeddings(canvasIds, selfConditioningLogits, opts) {
|
|
1209
|
+
const config = this.#state.modelConfig;
|
|
1210
|
+
const canvasLength = canvasIds.length;
|
|
1211
|
+
const hiddenSize = config.hiddenSize;
|
|
1212
|
+
const vocabSize = config.vocabSize;
|
|
1213
|
+
const elementCount = canvasLength * hiddenSize;
|
|
1214
|
+
const softEmbeddingState = normalizeSelfConditioningSoftEmbeddingState(
|
|
1215
|
+
selfConditioningLogits,
|
|
1216
|
+
canvasLength,
|
|
1217
|
+
hiddenSize
|
|
1218
|
+
);
|
|
1219
|
+
const logitsState = softEmbeddingState
|
|
1220
|
+
? null
|
|
1221
|
+
: normalizeSelfConditioningLogitsState(selfConditioningLogits, canvasLength, vocabSize);
|
|
1222
|
+
const logits = (softEmbeddingState || logitsState)
|
|
1223
|
+
? null
|
|
1224
|
+
: normalizeSelfConditioningLogits(selfConditioningLogits, canvasLength, vocabSize);
|
|
1225
|
+
const weights = this.#state.weights.get('diffusion_gemma_self_conditioning');
|
|
1226
|
+
if (!weights || typeof weights !== 'object') {
|
|
1227
|
+
throw new Error(
|
|
1228
|
+
'DiffusionGemma self-conditioning weights were not loaded. ' +
|
|
1229
|
+
'Expected model.decoder.self_conditioning tensors in the manifest.'
|
|
1230
|
+
);
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
const embedBufferRaw = this.#state.weights.get('embed');
|
|
1234
|
+
if (!embedBufferRaw) {
|
|
1235
|
+
throw new Error('DiffusionGemma self-conditioning requires loaded embed_tokens weights.');
|
|
1236
|
+
}
|
|
1237
|
+
const borrowed = {
|
|
1238
|
+
preNorm: null,
|
|
1239
|
+
gateProj: null,
|
|
1240
|
+
upProj: null,
|
|
1241
|
+
downProj: null,
|
|
1242
|
+
softEmbedding: null,
|
|
1243
|
+
};
|
|
1244
|
+
let softLogitsTensor = null;
|
|
1245
|
+
let softLogitsOwned = true;
|
|
1246
|
+
let softEmbeddingStateTensor = null;
|
|
1247
|
+
let softEmbeddingStateOwned = false;
|
|
1248
|
+
let softmaxTensor = null;
|
|
1249
|
+
let softEmbeddings = null;
|
|
1250
|
+
let scaledSoftEmbeddings = null;
|
|
1251
|
+
let baseEmbeddings = null;
|
|
1252
|
+
let normedSoft = null;
|
|
1253
|
+
let gate = null;
|
|
1254
|
+
let up = null;
|
|
1255
|
+
let activated = null;
|
|
1256
|
+
let down = null;
|
|
1257
|
+
let combined = null;
|
|
1258
|
+
let output = null;
|
|
1259
|
+
|
|
1260
|
+
const releaseTensorOnce = (() => {
|
|
1261
|
+
const released = new Set();
|
|
1262
|
+
return (tensor) => {
|
|
1263
|
+
const buffer = tensor?.buffer ?? null;
|
|
1264
|
+
if (!buffer || released.has(buffer)) return;
|
|
1265
|
+
released.add(buffer);
|
|
1266
|
+
releaseBuffer(buffer);
|
|
1267
|
+
};
|
|
1268
|
+
})();
|
|
1269
|
+
|
|
1270
|
+
try {
|
|
1271
|
+
borrowed.preNorm = borrowNormWeight(weights.preNorm, 'diffusion_gemma_self_conditioning.pre_norm');
|
|
1272
|
+
borrowed.gateProj = borrowLinearWeight(weights.gateProj, 'diffusion_gemma_self_conditioning.gate_proj');
|
|
1273
|
+
borrowed.upProj = borrowLinearWeight(weights.upProj, 'diffusion_gemma_self_conditioning.up_proj');
|
|
1274
|
+
borrowed.downProj = borrowLinearWeight(weights.downProj, 'diffusion_gemma_self_conditioning.down_proj');
|
|
1275
|
+
if (isSplitWeightBuffer(embedBufferRaw) && this.#state.embeddingTranspose === true) {
|
|
1276
|
+
throw new Error(
|
|
1277
|
+
'DiffusionGemma self-conditioning split embeddings require row-major embedding storage.'
|
|
1278
|
+
);
|
|
1279
|
+
}
|
|
1280
|
+
borrowed.softEmbedding = isSplitWeightBuffer(embedBufferRaw)
|
|
1281
|
+
? null
|
|
1282
|
+
: borrowLinearWeight(embedBufferRaw, 'diffusion_gemma_self_conditioning.embed_tokens');
|
|
1283
|
+
|
|
1284
|
+
const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
|
|
1285
|
+
const embedDtype = isCpuWeightBuffer(embedBufferRaw)
|
|
1286
|
+
? embedBufferRaw.dtype
|
|
1287
|
+
: getWeightDtype(embedBufferRaw);
|
|
1288
|
+
const embedMetadata = getWeightMetadata(embedBufferRaw);
|
|
1289
|
+
baseEmbeddings = await embed(canvasIds, embedBuffer, {
|
|
1290
|
+
hiddenSize,
|
|
1291
|
+
vocabSize,
|
|
1292
|
+
scaleEmbeddings: config.scaleEmbeddings,
|
|
1293
|
+
debug: opts.debug,
|
|
1294
|
+
recorder: null,
|
|
1295
|
+
transpose: this.#state.embeddingTranspose,
|
|
1296
|
+
activationDtype: 'f32',
|
|
1297
|
+
embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
|
|
1298
|
+
embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
|
|
1299
|
+
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1300
|
+
operatorDiagnostics: this.#state.operatorDiagnostics,
|
|
1301
|
+
});
|
|
1302
|
+
|
|
1303
|
+
if (softEmbeddingState) {
|
|
1304
|
+
softEmbeddingStateOwned = softEmbeddingState.releaseOnUse;
|
|
1305
|
+
softEmbeddings = createTensor(
|
|
1306
|
+
softEmbeddingState.buffer,
|
|
1307
|
+
softEmbeddingState.dtype,
|
|
1308
|
+
[canvasLength, hiddenSize],
|
|
1309
|
+
'diffusion_gemma_self_conditioning_soft_embedding'
|
|
1310
|
+
);
|
|
1311
|
+
softEmbeddingStateTensor = softEmbeddings;
|
|
1312
|
+
if (!softEmbeddingState.scaled) {
|
|
1313
|
+
scaledSoftEmbeddings = await runScale(softEmbeddings, Math.sqrt(hiddenSize), {
|
|
1314
|
+
count: elementCount,
|
|
1315
|
+
});
|
|
1316
|
+
if (softEmbeddingStateOwned) {
|
|
1317
|
+
releaseTensorOnce(softEmbeddings);
|
|
1318
|
+
}
|
|
1319
|
+
softEmbeddings = scaledSoftEmbeddings;
|
|
1320
|
+
softEmbeddingStateTensor = null;
|
|
1321
|
+
scaledSoftEmbeddings = null;
|
|
1322
|
+
}
|
|
1323
|
+
} else if (logitsState || logits) {
|
|
1324
|
+
const softmaxTemperature = logitsState?.temperature ?? 1.0;
|
|
1325
|
+
if (logitsState) {
|
|
1326
|
+
softLogitsOwned = logitsState.releaseOnUse;
|
|
1327
|
+
softLogitsTensor = createTensor(
|
|
1328
|
+
logitsState.logitsBuffer,
|
|
1329
|
+
logitsState.logitsDtype,
|
|
1330
|
+
[canvasLength, vocabSize],
|
|
1331
|
+
'diffusion_gemma_self_conditioning_logits'
|
|
1332
|
+
);
|
|
1333
|
+
} else {
|
|
1334
|
+
const logitsBuffer = acquireBuffer(logits.byteLength, undefined, 'diffusion_gemma_self_conditioning_logits');
|
|
1335
|
+
uploadData(logitsBuffer, logits);
|
|
1336
|
+
softLogitsTensor = createTensor(logitsBuffer, 'f32', [canvasLength, vocabSize], 'diffusion_gemma_self_conditioning_logits');
|
|
1337
|
+
}
|
|
1338
|
+
if (canUseChunkedSoftEmbeddingLogits(
|
|
1339
|
+
logitsState,
|
|
1340
|
+
borrowed.softEmbedding?.value,
|
|
1341
|
+
this.#state.embeddingTranspose
|
|
1342
|
+
)) {
|
|
1343
|
+
softEmbeddings = await runSoftEmbeddingLogitsF16(
|
|
1344
|
+
softLogitsTensor,
|
|
1345
|
+
borrowed.softEmbedding.value,
|
|
1346
|
+
canvasLength,
|
|
1347
|
+
hiddenSize,
|
|
1348
|
+
vocabSize,
|
|
1349
|
+
{
|
|
1350
|
+
temperature: softmaxTemperature,
|
|
1351
|
+
chunkRows: resolveDiffusionGemmaSoftEmbeddingChunkRows(this.#state.runtimeConfig),
|
|
1352
|
+
}
|
|
1353
|
+
);
|
|
1354
|
+
} else {
|
|
1355
|
+
softmaxTensor = await runSoftmax(softLogitsTensor, -1, {
|
|
1356
|
+
batchSize: canvasLength,
|
|
1357
|
+
size: vocabSize,
|
|
1358
|
+
temperature: softmaxTemperature,
|
|
1359
|
+
});
|
|
1360
|
+
softEmbeddings = isSplitWeightBuffer(embedBufferRaw)
|
|
1361
|
+
? await runSoftEmbeddingSplitF16(
|
|
1362
|
+
softmaxTensor,
|
|
1363
|
+
embedBufferRaw,
|
|
1364
|
+
canvasLength,
|
|
1365
|
+
hiddenSize,
|
|
1366
|
+
vocabSize
|
|
1367
|
+
)
|
|
1368
|
+
: await runMatmul(
|
|
1369
|
+
softmaxTensor,
|
|
1370
|
+
borrowed.softEmbedding.value,
|
|
1371
|
+
canvasLength,
|
|
1372
|
+
hiddenSize,
|
|
1373
|
+
vocabSize,
|
|
1374
|
+
{
|
|
1375
|
+
transposeB: this.#state.embeddingTranspose === true,
|
|
1376
|
+
role: 'diffusion_gemma_self_conditioning_embed',
|
|
1377
|
+
outputDtype: 'f32',
|
|
1378
|
+
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1379
|
+
}
|
|
1380
|
+
);
|
|
1381
|
+
}
|
|
1382
|
+
scaledSoftEmbeddings = await runScale(softEmbeddings, Math.sqrt(hiddenSize), {
|
|
1383
|
+
count: elementCount,
|
|
1384
|
+
});
|
|
1385
|
+
if (softEmbeddings !== softEmbeddingStateTensor || softEmbeddingStateOwned) {
|
|
1386
|
+
releaseTensorOnce(softEmbeddings);
|
|
1387
|
+
}
|
|
1388
|
+
softEmbeddings = scaledSoftEmbeddings;
|
|
1389
|
+
scaledSoftEmbeddings = null;
|
|
1390
|
+
} else {
|
|
1391
|
+
const zeroBuffer = acquireBuffer(elementCount * Float32Array.BYTES_PER_ELEMENT, undefined, 'diffusion_gemma_self_conditioning_zero');
|
|
1392
|
+
uploadData(zeroBuffer, new Uint8Array(elementCount * Float32Array.BYTES_PER_ELEMENT));
|
|
1393
|
+
softEmbeddings = createTensor(zeroBuffer, 'f32', [canvasLength, hiddenSize], 'diffusion_gemma_self_conditioning_zero');
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
normedSoft = await runRMSNorm(softEmbeddings, borrowed.preNorm.value, config.rmsNormEps, {
|
|
1397
|
+
batchSize: canvasLength,
|
|
1398
|
+
hiddenSize,
|
|
1399
|
+
rmsNormWeightOffset: false,
|
|
1400
|
+
});
|
|
1401
|
+
const intermediateSize = config.intermediateSize;
|
|
1402
|
+
gate = await runMatmul(
|
|
1403
|
+
normedSoft,
|
|
1404
|
+
borrowed.gateProj.value,
|
|
1405
|
+
canvasLength,
|
|
1406
|
+
intermediateSize,
|
|
1407
|
+
hiddenSize,
|
|
1408
|
+
{
|
|
1409
|
+
transposeB: 'auto',
|
|
1410
|
+
role: 'diffusion_gemma_self_conditioning_gate',
|
|
1411
|
+
outputDtype: 'f32',
|
|
1412
|
+
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1413
|
+
}
|
|
1414
|
+
);
|
|
1415
|
+
up = await runMatmul(
|
|
1416
|
+
normedSoft,
|
|
1417
|
+
borrowed.upProj.value,
|
|
1418
|
+
canvasLength,
|
|
1419
|
+
intermediateSize,
|
|
1420
|
+
hiddenSize,
|
|
1421
|
+
{
|
|
1422
|
+
transposeB: 'auto',
|
|
1423
|
+
role: 'diffusion_gemma_self_conditioning_up',
|
|
1424
|
+
outputDtype: 'f32',
|
|
1425
|
+
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1426
|
+
}
|
|
1427
|
+
);
|
|
1428
|
+
activated = await runGeLU(up, {
|
|
1429
|
+
size: canvasLength * intermediateSize,
|
|
1430
|
+
gate,
|
|
1431
|
+
});
|
|
1432
|
+
down = await runMatmul(
|
|
1433
|
+
activated,
|
|
1434
|
+
borrowed.downProj.value,
|
|
1435
|
+
canvasLength,
|
|
1436
|
+
hiddenSize,
|
|
1437
|
+
intermediateSize,
|
|
1438
|
+
{
|
|
1439
|
+
transposeB: 'auto',
|
|
1440
|
+
role: 'diffusion_gemma_self_conditioning_down',
|
|
1441
|
+
outputDtype: 'f32',
|
|
1442
|
+
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1443
|
+
}
|
|
1444
|
+
);
|
|
1445
|
+
combined = await runResidualAdd(baseEmbeddings, down, elementCount, {
|
|
1446
|
+
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1447
|
+
});
|
|
1448
|
+
const postNormWeight = weights.postNorm
|
|
1449
|
+
? borrowNormWeight(weights.postNorm, 'diffusion_gemma_self_conditioning.post_norm')
|
|
1450
|
+
: {
|
|
1451
|
+
value: getQKNormOnesBuffer(hiddenSize),
|
|
1452
|
+
owned: false,
|
|
1453
|
+
};
|
|
1454
|
+
borrowed.postNorm = postNormWeight;
|
|
1455
|
+
output = await runRMSNorm(combined, postNormWeight.value, config.rmsNormEps, {
|
|
1456
|
+
batchSize: canvasLength,
|
|
1457
|
+
hiddenSize,
|
|
1458
|
+
rmsNormWeightOffset: false,
|
|
1459
|
+
});
|
|
1460
|
+
return output;
|
|
1461
|
+
} catch (error) {
|
|
1462
|
+
releaseTensorOnce(output);
|
|
1463
|
+
throw error;
|
|
1464
|
+
} finally {
|
|
1465
|
+
releaseTensorOnce(combined);
|
|
1466
|
+
releaseTensorOnce(down);
|
|
1467
|
+
releaseTensorOnce(activated);
|
|
1468
|
+
releaseTensorOnce(up);
|
|
1469
|
+
releaseTensorOnce(gate);
|
|
1470
|
+
releaseTensorOnce(normedSoft);
|
|
1471
|
+
if (softEmbeddings !== softEmbeddingStateTensor || softEmbeddingStateOwned) {
|
|
1472
|
+
releaseTensorOnce(softEmbeddings);
|
|
1473
|
+
}
|
|
1474
|
+
releaseTensorOnce(softmaxTensor);
|
|
1475
|
+
if (softLogitsOwned) {
|
|
1476
|
+
releaseTensorOnce(softLogitsTensor);
|
|
1477
|
+
}
|
|
1478
|
+
releaseTensorOnce(baseEmbeddings);
|
|
1479
|
+
releaseBorrowedWeight(borrowed.postNorm);
|
|
1480
|
+
releaseBorrowedWeight(borrowed.downProj);
|
|
1481
|
+
releaseBorrowedWeight(borrowed.upProj);
|
|
1482
|
+
releaseBorrowedWeight(borrowed.gateProj);
|
|
1483
|
+
releaseBorrowedWeight(borrowed.preNorm);
|
|
1484
|
+
releaseBorrowedWeight(borrowed.softEmbedding);
|
|
1485
|
+
}
|
|
1486
|
+
}
|
|
1487
|
+
|
|
1488
|
+
async _createDiffusionGemmaSelfConditioningSoftEmbeddingState(logitsState, canvasLength, hiddenSize, vocabSize) {
|
|
1489
|
+
const embedBufferRaw = this.#state.weights.get('embed');
|
|
1490
|
+
if (!canUseChunkedSoftEmbeddingLogits(
|
|
1491
|
+
logitsState,
|
|
1492
|
+
embedBufferRaw,
|
|
1493
|
+
this.#state.embeddingTranspose
|
|
1494
|
+
)) {
|
|
1495
|
+
return null;
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
const borrowed = borrowLinearWeight(embedBufferRaw, 'diffusion_gemma_self_conditioning.embed_tokens');
|
|
1499
|
+
let softEmbedding = null;
|
|
1500
|
+
let scaledSoftEmbedding = null;
|
|
1501
|
+
try {
|
|
1502
|
+
const logitsTensor = createTensor(
|
|
1503
|
+
logitsState.logitsBuffer,
|
|
1504
|
+
logitsState.logitsDtype,
|
|
1505
|
+
[canvasLength, vocabSize],
|
|
1506
|
+
'diffusion_gemma_self_conditioning_logits'
|
|
1507
|
+
);
|
|
1508
|
+
softEmbedding = await runSoftEmbeddingLogitsF16(
|
|
1509
|
+
logitsTensor,
|
|
1510
|
+
borrowed.value,
|
|
1511
|
+
canvasLength,
|
|
1512
|
+
hiddenSize,
|
|
1513
|
+
vocabSize,
|
|
1514
|
+
{
|
|
1515
|
+
temperature: logitsState.temperature,
|
|
1516
|
+
chunkRows: resolveDiffusionGemmaSoftEmbeddingChunkRows(this.#state.runtimeConfig),
|
|
1517
|
+
}
|
|
1518
|
+
);
|
|
1519
|
+
scaledSoftEmbedding = await runScale(softEmbedding, Math.sqrt(hiddenSize), {
|
|
1520
|
+
count: canvasLength * hiddenSize,
|
|
1521
|
+
});
|
|
1522
|
+
const returnedBuffer = scaledSoftEmbedding.buffer;
|
|
1523
|
+
return {
|
|
1524
|
+
kind: 'soft_embedding',
|
|
1525
|
+
buffer: returnedBuffer,
|
|
1526
|
+
dtype: scaledSoftEmbedding.dtype,
|
|
1527
|
+
canvasLength,
|
|
1528
|
+
hiddenSize,
|
|
1529
|
+
scaled: true,
|
|
1530
|
+
releaseOnUse: true,
|
|
1531
|
+
release() {
|
|
1532
|
+
releaseBuffer(returnedBuffer);
|
|
1533
|
+
},
|
|
1534
|
+
};
|
|
1535
|
+
} finally {
|
|
1536
|
+
if (softEmbedding?.buffer && softEmbedding.buffer !== scaledSoftEmbedding?.buffer) {
|
|
1537
|
+
releaseBuffer(softEmbedding.buffer);
|
|
1538
|
+
}
|
|
1539
|
+
releaseBorrowedWeight(borrowed);
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
|
|
1543
|
+
async computeDiffusionGemmaCanvasLogits(args, options = {}) {
|
|
1544
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
1545
|
+
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
1546
|
+
throw new Error('Generation already in progress');
|
|
1547
|
+
}
|
|
1548
|
+
const canvasIds = normalizeCanvasTokenIds(args?.canvas, 'computeDiffusionGemmaCanvasLogits');
|
|
1549
|
+
if (canvasIds.length === 0) {
|
|
1550
|
+
throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasLogits requires at least one canvas token.');
|
|
1551
|
+
}
|
|
1552
|
+
this._assertTokenIdsInRange(canvasIds, 'computeDiffusionGemmaCanvasLogits.canvas');
|
|
1553
|
+
const seqLenBefore = this.#state.currentSeqLen;
|
|
1554
|
+
const opts = {
|
|
1555
|
+
...resolvePrefillOptions(this.#state, {
|
|
1556
|
+
...options,
|
|
1557
|
+
useChatTemplate: false,
|
|
1558
|
+
}),
|
|
1559
|
+
_diffusionGemmaDecoder: true,
|
|
1560
|
+
};
|
|
1561
|
+
let selfConditioned = null;
|
|
1562
|
+
let currentHiddenBuffer = null;
|
|
1563
|
+
try {
|
|
1564
|
+
selfConditioned = await this._createDiffusionGemmaSelfConditioningEmbeddings(
|
|
1565
|
+
canvasIds,
|
|
1566
|
+
args?.selfConditioningLogits ?? null,
|
|
1567
|
+
opts
|
|
1568
|
+
);
|
|
1569
|
+
const prefillResult = await this._prefillToHidden(canvasIds, {
|
|
1570
|
+
...opts,
|
|
1571
|
+
embeddingOverrides: {
|
|
1572
|
+
offset: 0,
|
|
1573
|
+
prefixLength: canvasIds.length,
|
|
1574
|
+
embeddings: selfConditioned.buffer,
|
|
1575
|
+
},
|
|
1576
|
+
});
|
|
1577
|
+
const {
|
|
1578
|
+
numTokens,
|
|
1579
|
+
currentRecorder,
|
|
1580
|
+
recordProfile,
|
|
1581
|
+
debugCheckBuffer,
|
|
1582
|
+
} = prefillResult;
|
|
1583
|
+
currentHiddenBuffer = prefillResult.currentHiddenBuffer;
|
|
1584
|
+
if (currentRecorder) {
|
|
1585
|
+
await currentRecorder.submitAndWait();
|
|
1586
|
+
await recordProfile(currentRecorder);
|
|
1587
|
+
} else {
|
|
1588
|
+
const device = getDevice();
|
|
1589
|
+
if (device) {
|
|
1590
|
+
await device.queue.onSubmittedWorkDone();
|
|
1591
|
+
}
|
|
1592
|
+
}
|
|
1593
|
+
const logits = await computeLogits(
|
|
1594
|
+
currentHiddenBuffer,
|
|
1595
|
+
numTokens,
|
|
1596
|
+
getLogitsWeights(this.#state),
|
|
1597
|
+
getLogitsConfig(this.#state),
|
|
1598
|
+
this.#state.useGPU,
|
|
1599
|
+
this.#state.debugFlags,
|
|
1600
|
+
undefined,
|
|
1601
|
+
debugCheckBuffer,
|
|
1602
|
+
this.#state.runtimeConfig.shared.debug.probes,
|
|
1603
|
+
{ lastPositionOnly: false },
|
|
1604
|
+
this.#state.operatorDiagnostics
|
|
1605
|
+
);
|
|
1606
|
+
const expected = canvasIds.length * this.#state.modelConfig.vocabSize;
|
|
1607
|
+
if (logits.length !== expected) {
|
|
1608
|
+
throw new Error(
|
|
1609
|
+
`[DiffusionGemma] canvas logits length mismatch: expected ${expected}, got ${logits.length}.`
|
|
1610
|
+
);
|
|
1611
|
+
}
|
|
1612
|
+
return logits;
|
|
1613
|
+
} finally {
|
|
1614
|
+
this.#state.currentSeqLen = seqLenBefore;
|
|
1615
|
+
if (currentHiddenBuffer) {
|
|
1616
|
+
releaseBuffer(currentHiddenBuffer);
|
|
1617
|
+
}
|
|
1618
|
+
if (selfConditioned?.buffer) {
|
|
1619
|
+
releaseBuffer(selfConditioned.buffer);
|
|
1620
|
+
}
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
1623
|
+
|
|
1624
|
+
async computeDiffusionGemmaCanvasStep(args, options = {}) {
|
|
1625
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
1626
|
+
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
1627
|
+
throw new Error('Generation already in progress');
|
|
1628
|
+
}
|
|
1629
|
+
const canvasIds = normalizeCanvasTokenIds(args?.canvas, 'computeDiffusionGemmaCanvasStep');
|
|
1630
|
+
if (canvasIds.length === 0) {
|
|
1631
|
+
throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasStep requires at least one canvas token.');
|
|
1632
|
+
}
|
|
1633
|
+
this._assertTokenIdsInRange(canvasIds, 'computeDiffusionGemmaCanvasStep.canvas');
|
|
1634
|
+
const temperature = args?.temperature;
|
|
1635
|
+
if (typeof temperature !== 'number' || !Number.isFinite(temperature) || temperature <= 0) {
|
|
1636
|
+
throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasStep requires a positive temperature.');
|
|
1637
|
+
}
|
|
1638
|
+
const seqLenBefore = this.#state.currentSeqLen;
|
|
1639
|
+
const opts = {
|
|
1640
|
+
...resolvePrefillOptions(this.#state, {
|
|
1641
|
+
...options,
|
|
1642
|
+
useChatTemplate: false,
|
|
1643
|
+
}),
|
|
1644
|
+
_diffusionGemmaDecoder: true,
|
|
1645
|
+
};
|
|
1646
|
+
let selfConditioned = null;
|
|
1647
|
+
let currentHiddenBuffer = null;
|
|
1648
|
+
let logitsBuffer = null;
|
|
1649
|
+
let statsBuffers = null;
|
|
1650
|
+
try {
|
|
1651
|
+
selfConditioned = await this._createDiffusionGemmaSelfConditioningEmbeddings(
|
|
1652
|
+
canvasIds,
|
|
1653
|
+
args?.selfConditioningLogits ?? null,
|
|
1654
|
+
opts
|
|
1655
|
+
);
|
|
1656
|
+
const prefillResult = await this._prefillToHidden(canvasIds, {
|
|
1657
|
+
...opts,
|
|
1658
|
+
embeddingOverrides: {
|
|
1659
|
+
offset: 0,
|
|
1660
|
+
prefixLength: canvasIds.length,
|
|
1661
|
+
embeddings: selfConditioned.buffer,
|
|
1662
|
+
},
|
|
1663
|
+
});
|
|
1664
|
+
const {
|
|
1665
|
+
numTokens,
|
|
1666
|
+
currentRecorder,
|
|
1667
|
+
recordProfile,
|
|
1668
|
+
debugCheckBuffer,
|
|
1669
|
+
} = prefillResult;
|
|
1670
|
+
currentHiddenBuffer = prefillResult.currentHiddenBuffer;
|
|
1671
|
+
if (currentRecorder) {
|
|
1672
|
+
await currentRecorder.submitAndWait();
|
|
1673
|
+
await recordProfile(currentRecorder);
|
|
1674
|
+
} else {
|
|
1675
|
+
const device = getDevice();
|
|
1676
|
+
if (device) {
|
|
1677
|
+
await device.queue.onSubmittedWorkDone();
|
|
1678
|
+
}
|
|
1679
|
+
}
|
|
1680
|
+
if (selfConditioned?.buffer) {
|
|
1681
|
+
releaseBuffer(selfConditioned.buffer);
|
|
1682
|
+
selfConditioned = null;
|
|
1683
|
+
}
|
|
1684
|
+
|
|
1685
|
+
const logitsResult = await computeLogitsGPU(
|
|
1686
|
+
currentHiddenBuffer,
|
|
1687
|
+
numTokens,
|
|
1688
|
+
getLogitsWeights(this.#state),
|
|
1689
|
+
getLogitsConfig(this.#state),
|
|
1690
|
+
this.#state.debugFlags,
|
|
1691
|
+
this.#state.operatorDiagnostics
|
|
1692
|
+
);
|
|
1693
|
+
if (!logitsResult?.logitsBuffer) {
|
|
1694
|
+
throw new Error('[DiffusionGemma] GPU canvas step requires GPU logits.');
|
|
1695
|
+
}
|
|
1696
|
+
logitsBuffer = logitsResult.logitsBuffer;
|
|
1697
|
+
if (currentHiddenBuffer) {
|
|
1698
|
+
releaseBuffer(currentHiddenBuffer);
|
|
1699
|
+
currentHiddenBuffer = null;
|
|
1700
|
+
}
|
|
1701
|
+
if (logitsResult.logitsDtype !== 'f32') {
|
|
1702
|
+
throw new Error(
|
|
1703
|
+
`[DiffusionGemma] GPU canvas stats require f32 logits, got "${logitsResult.logitsDtype}".`
|
|
1704
|
+
);
|
|
1705
|
+
}
|
|
1706
|
+
if (logitsResult.vocabSize !== this.#state.modelConfig.vocabSize) {
|
|
1707
|
+
throw new Error(
|
|
1708
|
+
`[DiffusionGemma] canvas logits vocab mismatch: expected ${this.#state.modelConfig.vocabSize}, ` +
|
|
1709
|
+
`got ${logitsResult.vocabSize}.`
|
|
1710
|
+
);
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
statsBuffers = await runDiffusionGemmaCanvasStats(logitsBuffer, {
|
|
1714
|
+
canvasLength: canvasIds.length,
|
|
1715
|
+
vocabSize: logitsResult.vocabSize,
|
|
1716
|
+
temperature,
|
|
1717
|
+
padTokenId: this.#state.modelConfig.diffusionGemma?.padTokenId ?? null,
|
|
1718
|
+
logitSoftcap: this.#state.modelConfig.finalLogitSoftcapping ?? 0,
|
|
1719
|
+
});
|
|
1720
|
+
const [argmaxData, entropyData] = await Promise.all([
|
|
1721
|
+
readBuffer(statsBuffers.argmaxBuffer, canvasIds.length * Uint32Array.BYTES_PER_ELEMENT),
|
|
1722
|
+
readBuffer(statsBuffers.entropyBuffer, canvasIds.length * Float32Array.BYTES_PER_ELEMENT),
|
|
1723
|
+
]);
|
|
1724
|
+
const argmaxCanvas = Int32Array.from(new Uint32Array(argmaxData));
|
|
1725
|
+
const entropies = new Float32Array(entropyData);
|
|
1726
|
+
const logitsState = {
|
|
1727
|
+
logitsBuffer,
|
|
1728
|
+
logitsDtype: logitsResult.logitsDtype,
|
|
1729
|
+
vocabSize: logitsResult.vocabSize,
|
|
1730
|
+
canvasLength: canvasIds.length,
|
|
1731
|
+
temperature,
|
|
1732
|
+
releaseOnUse: true,
|
|
1733
|
+
};
|
|
1734
|
+
const selfConditioningState =
|
|
1735
|
+
await this._createDiffusionGemmaSelfConditioningSoftEmbeddingState(
|
|
1736
|
+
logitsState,
|
|
1737
|
+
canvasIds.length,
|
|
1738
|
+
this.#state.modelConfig.hiddenSize,
|
|
1739
|
+
logitsResult.vocabSize
|
|
1740
|
+
);
|
|
1741
|
+
if (selfConditioningState) {
|
|
1742
|
+
releaseBuffer(logitsBuffer);
|
|
1743
|
+
logitsBuffer = null;
|
|
1744
|
+
} else {
|
|
1745
|
+
const returnedLogitsBuffer = logitsBuffer;
|
|
1746
|
+
logitsBuffer = null;
|
|
1747
|
+
logitsState.release = () => {
|
|
1748
|
+
releaseBuffer(returnedLogitsBuffer);
|
|
1749
|
+
};
|
|
1750
|
+
}
|
|
1751
|
+
return {
|
|
1752
|
+
argmaxCanvas,
|
|
1753
|
+
entropies,
|
|
1754
|
+
selfConditioningLogits: selfConditioningState ?? logitsState,
|
|
1755
|
+
};
|
|
1756
|
+
} finally {
|
|
1757
|
+
this.#state.currentSeqLen = seqLenBefore;
|
|
1758
|
+
if (statsBuffers?.argmaxBuffer) {
|
|
1759
|
+
releaseBuffer(statsBuffers.argmaxBuffer);
|
|
1760
|
+
}
|
|
1761
|
+
if (statsBuffers?.entropyBuffer) {
|
|
1762
|
+
releaseBuffer(statsBuffers.entropyBuffer);
|
|
1763
|
+
}
|
|
1764
|
+
if (logitsBuffer) {
|
|
1765
|
+
releaseBuffer(logitsBuffer);
|
|
1766
|
+
}
|
|
1767
|
+
if (currentHiddenBuffer) {
|
|
1768
|
+
releaseBuffer(currentHiddenBuffer);
|
|
1769
|
+
}
|
|
1770
|
+
if (selfConditioned?.buffer) {
|
|
1771
|
+
releaseBuffer(selfConditioned.buffer);
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
1774
|
+
}
|
|
1775
|
+
|
|
853
1776
|
async *generate(prompt, options = {}) {
|
|
854
1777
|
yield* this._generateTokensInternal(prompt, options, 'text');
|
|
855
1778
|
}
|
|
@@ -869,12 +1792,21 @@ export class PipelineGenerator {
|
|
|
869
1792
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
870
1793
|
this.#state.stats.gpuTimeDecodeMs = undefined;
|
|
871
1794
|
this.#state.stats.decodeRecordMs = 0;
|
|
1795
|
+
this.#state.stats.decodeRecordOps = 0;
|
|
1796
|
+
this.#state.stats.decodeRecordPasses = 0;
|
|
1797
|
+
this.#state.stats.decodeRecordOpLabels = {};
|
|
872
1798
|
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
873
1799
|
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
1800
|
+
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
1801
|
+
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
1802
|
+
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
874
1803
|
this.#state.stats.prefillRecordMs = 0;
|
|
875
1804
|
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
876
1805
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
877
1806
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
1807
|
+
this.#state.stats.singleTokenReadbackMapWaitMs = 0;
|
|
1808
|
+
this.#state.stats.singleTokenReadbackCleanupMs = 0;
|
|
1809
|
+
this.#state.stats.singleTokenReadbackCopyMs = 0;
|
|
878
1810
|
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
879
1811
|
this.#state.stats.pleHotVocabularyHits = 0;
|
|
880
1812
|
this.#state.stats.pleHotVocabularyMisses = 0;
|
|
@@ -896,6 +1828,7 @@ export class PipelineGenerator {
|
|
|
896
1828
|
opts.topP = samplingConfig.topP;
|
|
897
1829
|
opts.topK = samplingConfig.topK;
|
|
898
1830
|
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
1831
|
+
opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this.#state, samplingConfig);
|
|
899
1832
|
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
900
1833
|
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
901
1834
|
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
@@ -1262,8 +2195,14 @@ export class PipelineGenerator {
|
|
|
1262
2195
|
this.#state.stats.prefillProfileSteps = [];
|
|
1263
2196
|
this.#state.decodeRing?.reset();
|
|
1264
2197
|
this.#state.stats.decodeRecordMs = 0;
|
|
2198
|
+
this.#state.stats.decodeRecordOps = 0;
|
|
2199
|
+
this.#state.stats.decodeRecordPasses = 0;
|
|
2200
|
+
this.#state.stats.decodeRecordOpLabels = {};
|
|
1265
2201
|
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
1266
2202
|
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
2203
|
+
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
2204
|
+
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
2205
|
+
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
1267
2206
|
this.#state.stats.ttftMs = 0;
|
|
1268
2207
|
const startTime = performance.now();
|
|
1269
2208
|
|
|
@@ -1290,6 +2229,7 @@ export class PipelineGenerator {
|
|
|
1290
2229
|
topK: opts.topK,
|
|
1291
2230
|
padTokenId,
|
|
1292
2231
|
seed: opts.seed,
|
|
2232
|
+
suppressTokenIds: opts.suppressTokenIds,
|
|
1293
2233
|
});
|
|
1294
2234
|
|
|
1295
2235
|
const decodeRuntime = {
|
|
@@ -1372,6 +2312,7 @@ export class PipelineGenerator {
|
|
|
1372
2312
|
const hasLinearLayers = hasLinearAttentionLayers(this.#state.modelConfig.layerTypes);
|
|
1373
2313
|
const replayPrefillDecode = usesReplayPrefillDecode(this.#state);
|
|
1374
2314
|
const gpuSamplingAvailable = isGPUSamplingAvailable() && !hasCpuWeights;
|
|
2315
|
+
const hasSuppressedSamplingTokens = Array.isArray(opts.suppressTokenIds) && opts.suppressTokenIds.length > 0;
|
|
1375
2316
|
const hasRangeBackedPerLayerInputs = hasRangeBackedPerLayerInputEmbeddings({
|
|
1376
2317
|
config: this.#state.modelConfig,
|
|
1377
2318
|
weights: this.#state.weights,
|
|
@@ -1379,6 +2320,7 @@ export class PipelineGenerator {
|
|
|
1379
2320
|
const hasGpuSplitPerLayerInputs = hasGpuSplitPerLayerInputEmbeddings({
|
|
1380
2321
|
config: this.#state.modelConfig,
|
|
1381
2322
|
weights: this.#state.weights,
|
|
2323
|
+
perLayerInputsSession: resolvedPerLayerInputsSession,
|
|
1382
2324
|
});
|
|
1383
2325
|
const pleHotVocabularyRuntime = getPleHotVocabularyRuntime({ weights: this.#state.weights });
|
|
1384
2326
|
const resolveCurrentHotVocabularyBatchDecodeAvailable = () => resolveHotVocabularyBatchDecodeAvailability({
|
|
@@ -1394,7 +2336,7 @@ export class PipelineGenerator {
|
|
|
1394
2336
|
batchSize: executionPlan.batchSize,
|
|
1395
2337
|
useGPU: this.#state.useGPU,
|
|
1396
2338
|
gpuSamplingAvailable,
|
|
1397
|
-
disableMultiTokenDecode: executionPlan.disableMultiTokenDecode,
|
|
2339
|
+
disableMultiTokenDecode: executionPlan.disableMultiTokenDecode || hasSuppressedSamplingTokens,
|
|
1398
2340
|
disableCommandBatching: executionPlan.disableCommandBatching,
|
|
1399
2341
|
isBdpaPagedLayout: this.#state.kvCache?.layout === 'bdpa_paged',
|
|
1400
2342
|
finitenessFallbackWindowOpen: this._hasFinitenessFallbackWindow(),
|
|
@@ -1409,6 +2351,7 @@ export class PipelineGenerator {
|
|
|
1409
2351
|
else if (!this.#state.useGPU) reason = 'no_gpu';
|
|
1410
2352
|
else if (!gpuSamplingAvailable) reason = 'no_gpu_sampling';
|
|
1411
2353
|
else if (executionPlan.disableCommandBatching) reason = 'command_batching_disabled';
|
|
2354
|
+
else if (hasSuppressedSamplingTokens) reason = 'sampling_suppression_requires_cpu_logits';
|
|
1412
2355
|
else if (executionPlan.disableMultiTokenDecode) reason = 'multi_token_decode_disabled';
|
|
1413
2356
|
else if (executionPlan.batchSize <= 1) reason = 'batch_size_1';
|
|
1414
2357
|
else if (this.#state.kvCache?.layout === 'bdpa_paged') reason = 'bdpa_paged_layout';
|
|
@@ -1454,21 +2397,58 @@ export class PipelineGenerator {
|
|
|
1454
2397
|
hasHotVocabularyBatchDecode: hotVocabularyBatchDecodeAvailable,
|
|
1455
2398
|
hasGpuSplitPerLayerInputs,
|
|
1456
2399
|
hasLinearAttentionLayers: hasLinearLayers,
|
|
2400
|
+
modelId: this.#state.modelConfig.modelId ?? executionPlan.kernelPathId,
|
|
2401
|
+
activationDtype: executionPlan.activationDtype,
|
|
2402
|
+
currentSeqLen: this.#state.currentSeqLen,
|
|
2403
|
+
maxDecodeTokens: opts.maxTokens,
|
|
2404
|
+
numLayers: this.#state.modelConfig.numLayers,
|
|
2405
|
+
hiddenSize: this.#state.modelConfig.hiddenSize,
|
|
1457
2406
|
});
|
|
1458
2407
|
const requestedBatchTokens = executionPlan.batchSize * intervalBatches;
|
|
1459
2408
|
const boundedBatchTokens = maxBatchDecodeTokens == null
|
|
1460
2409
|
? requestedBatchTokens
|
|
1461
2410
|
: Math.min(requestedBatchTokens, maxBatchDecodeTokens);
|
|
1462
2411
|
const thisBatchSize = Math.min(boundedBatchTokens, remaining);
|
|
2412
|
+
this.#state.batchingStats.requestedBatchTokens = Math.max(
|
|
2413
|
+
this.#state.batchingStats.requestedBatchTokens ?? 0,
|
|
2414
|
+
requestedBatchTokens
|
|
2415
|
+
);
|
|
2416
|
+
this.#state.batchingStats.effectiveBatchTokens = Math.max(
|
|
2417
|
+
this.#state.batchingStats.effectiveBatchTokens ?? 0,
|
|
2418
|
+
thisBatchSize
|
|
2419
|
+
);
|
|
2420
|
+
if (maxBatchDecodeTokens != null) {
|
|
2421
|
+
this.#state.batchingStats.maxBatchTokenCap = Math.max(
|
|
2422
|
+
this.#state.batchingStats.maxBatchTokenCap ?? 0,
|
|
2423
|
+
maxBatchDecodeTokens
|
|
2424
|
+
);
|
|
2425
|
+
}
|
|
2426
|
+
if (thisBatchSize < requestedBatchTokens) {
|
|
2427
|
+
this.#state.batchingStats.batchClampCount = (this.#state.batchingStats.batchClampCount ?? 0) + 1;
|
|
2428
|
+
}
|
|
1463
2429
|
const lastToken = generatedIds[generatedIds.length - 1];
|
|
2430
|
+
const boundedExecutionPlan = thisBatchSize < requestedBatchTokens
|
|
2431
|
+
? {
|
|
2432
|
+
...executionPlan,
|
|
2433
|
+
batchSize: Math.min(executionPlan.batchSize, thisBatchSize),
|
|
2434
|
+
readbackInterval: readbackInterval == null
|
|
2435
|
+
? null
|
|
2436
|
+
: Math.min(intervalBatches, thisBatchSize),
|
|
2437
|
+
}
|
|
2438
|
+
: executionPlan;
|
|
2439
|
+
const batchOpts = boundedExecutionPlan === executionPlan
|
|
2440
|
+
? opts
|
|
2441
|
+
: { ...opts, executionPlan: boundedExecutionPlan };
|
|
1464
2442
|
|
|
1465
2443
|
try {
|
|
1466
|
-
const batchResult = await this._generateNTokensGPU(lastToken, thisBatchSize, generatedIds,
|
|
2444
|
+
const batchResult = await this._generateNTokensGPU(lastToken, thisBatchSize, generatedIds, batchOpts);
|
|
1467
2445
|
let batchTokens = [];
|
|
1468
2446
|
let hitStop = false;
|
|
2447
|
+
let stopTokenId = null;
|
|
1469
2448
|
for (const tokenId of batchResult.tokens) {
|
|
1470
2449
|
if (isStopToken(tokenId, stopTokenIds, eosToken)) {
|
|
1471
2450
|
hitStop = true;
|
|
2451
|
+
stopTokenId = tokenId;
|
|
1472
2452
|
break;
|
|
1473
2453
|
}
|
|
1474
2454
|
generatedIds.push(tokenId);
|
|
@@ -1497,7 +2477,7 @@ export class PipelineGenerator {
|
|
|
1497
2477
|
}
|
|
1498
2478
|
}
|
|
1499
2479
|
if (hitStop) {
|
|
1500
|
-
this._recordStopReason('stop-token');
|
|
2480
|
+
this._recordStopReason('stop-token', stopTokenId);
|
|
1501
2481
|
break;
|
|
1502
2482
|
}
|
|
1503
2483
|
if (shouldDisableBatchDecodeAfterShortBatch({
|
|
@@ -1721,6 +2701,7 @@ export class PipelineGenerator {
|
|
|
1721
2701
|
this.#state.stats.decodeTimeMs = performance.now() - decodeStart;
|
|
1722
2702
|
this.#state.stats.tokensGenerated = tokensGenerated;
|
|
1723
2703
|
this.#state.stats.decodeTokens = tokensGenerated;
|
|
2704
|
+
this.#state.stats.batching = { ...this.#state.batchingStats };
|
|
1724
2705
|
}
|
|
1725
2706
|
|
|
1726
2707
|
async _prefillToHidden(inputIds, opts) {
|
|
@@ -1733,6 +2714,8 @@ export class PipelineGenerator {
|
|
|
1733
2714
|
const numTokens = inputIds.length;
|
|
1734
2715
|
const config = this.#state.modelConfig;
|
|
1735
2716
|
const startPos = this.#state.currentSeqLen;
|
|
2717
|
+
const tracePrefillEnabled = isTraceEnabled('perf');
|
|
2718
|
+
const prefillTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
1736
2719
|
const returnHidden = opts?._returnHidden === true;
|
|
1737
2720
|
const embeddingInputIds = resolvePrefillEmbeddingInputIds(
|
|
1738
2721
|
inputIds,
|
|
@@ -1768,13 +2751,14 @@ export class PipelineGenerator {
|
|
|
1768
2751
|
}
|
|
1769
2752
|
|
|
1770
2753
|
const embedBufferRaw = this.#state.weights.get('embed');
|
|
1771
|
-
if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
|
|
2754
|
+
if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
|
|
1772
2755
|
throw new Error('Embed buffer not found or not a supported buffer type');
|
|
1773
2756
|
}
|
|
1774
2757
|
const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
|
|
1775
2758
|
const embedDtype = isCpuWeightBuffer(embedBufferRaw)
|
|
1776
2759
|
? embedBufferRaw.dtype
|
|
1777
2760
|
: getWeightDtype(embedBufferRaw);
|
|
2761
|
+
const embedMetadata = getWeightMetadata(embedBufferRaw);
|
|
1778
2762
|
if (opts.debug) {
|
|
1779
2763
|
const embedSize = isGpuBufferInstance(embedBuffer) ? embedBuffer.size : 'N/A';
|
|
1780
2764
|
log.debug('Pipeline', `Embed buffer: type=${embedBuffer?.constructor?.name}, size=${embedSize}, dtype=${embedDtype}`);
|
|
@@ -1789,7 +2773,9 @@ export class PipelineGenerator {
|
|
|
1789
2773
|
);
|
|
1790
2774
|
const createRecorder = (label) => {
|
|
1791
2775
|
if (!device || disableCommandBatching) return undefined;
|
|
1792
|
-
return opts.profile
|
|
2776
|
+
return opts.profile
|
|
2777
|
+
? createProfilingRecorder(label, device)
|
|
2778
|
+
: createCommandRecorder(label, { recordLabels: opts.debug === true }, device);
|
|
1793
2779
|
};
|
|
1794
2780
|
const recorder = createRecorder('prefill');
|
|
1795
2781
|
const debugCheckBuffer = this.#state.debug
|
|
@@ -1805,6 +2791,7 @@ export class PipelineGenerator {
|
|
|
1805
2791
|
opts.executionPlan
|
|
1806
2792
|
);
|
|
1807
2793
|
context.currentTokenIds = inputIds;
|
|
2794
|
+
context.diffusionGemmaDecoder = opts?._diffusionGemmaDecoder === true;
|
|
1808
2795
|
context.multimodalBidirectionalSpan = multimodalBidirectionalSpan == null
|
|
1809
2796
|
? null
|
|
1810
2797
|
: {
|
|
@@ -1854,6 +2841,16 @@ export class PipelineGenerator {
|
|
|
1854
2841
|
|
|
1855
2842
|
const activationDtype = opts.executionPlan?.activationDtype ?? this._getEffectiveActivationDtype();
|
|
1856
2843
|
const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
|
|
2844
|
+
if (tracePrefillEnabled) {
|
|
2845
|
+
trace.perf('Prefill phase start', {
|
|
2846
|
+
numTokens,
|
|
2847
|
+
startPos,
|
|
2848
|
+
numLayers: config.numLayers,
|
|
2849
|
+
activationDtype,
|
|
2850
|
+
returnHidden,
|
|
2851
|
+
});
|
|
2852
|
+
}
|
|
2853
|
+
const embedTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
1857
2854
|
let baseEmbeddings = await embed(embeddingInputIds, embedBuffer, {
|
|
1858
2855
|
hiddenSize: config.hiddenSize,
|
|
1859
2856
|
vocabSize: config.vocabSize,
|
|
@@ -1864,18 +2861,29 @@ export class PipelineGenerator {
|
|
|
1864
2861
|
debugProbes: this.#state.runtimeConfig.shared.debug.probes,
|
|
1865
2862
|
operatorDiagnostics: this.#state.operatorDiagnostics,
|
|
1866
2863
|
activationDtype,
|
|
1867
|
-
embeddingDtype: selectRuleValue('inference', 'dtype', '
|
|
2864
|
+
embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
|
|
2865
|
+
embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
|
|
1868
2866
|
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1869
2867
|
});
|
|
2868
|
+
if (tracePrefillEnabled) {
|
|
2869
|
+
trace.perf('Prefill embed complete', {
|
|
2870
|
+
numTokens,
|
|
2871
|
+
elapsedMs: performance.now() - embedTraceStart,
|
|
2872
|
+
});
|
|
2873
|
+
}
|
|
1870
2874
|
let hiddenStates = baseEmbeddings;
|
|
1871
2875
|
let perLayerInputs = null;
|
|
2876
|
+
const perLayerInputsTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
1872
2877
|
try {
|
|
1873
2878
|
hiddenStates = await applyPrefixEmbeddingOverride(
|
|
1874
2879
|
baseEmbeddings,
|
|
1875
2880
|
embeddingOverride,
|
|
1876
2881
|
config.hiddenSize,
|
|
1877
2882
|
'_prefill',
|
|
1878
|
-
|
|
2883
|
+
{
|
|
2884
|
+
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
2885
|
+
transitionDeclaredBy: resolvePrefixEmbeddingOverrideTransitionDeclaredBy(this.#state.executionV1State),
|
|
2886
|
+
}
|
|
1879
2887
|
);
|
|
1880
2888
|
perLayerInputs = await preparePerLayerInputs(
|
|
1881
2889
|
embeddingInputIds,
|
|
@@ -1886,6 +2894,13 @@ export class PipelineGenerator {
|
|
|
1886
2894
|
pleCache: this.#state.pleCache ?? null,
|
|
1887
2895
|
}
|
|
1888
2896
|
);
|
|
2897
|
+
if (tracePrefillEnabled) {
|
|
2898
|
+
trace.perf('Prefill per-layer inputs complete', {
|
|
2899
|
+
numTokens,
|
|
2900
|
+
elapsedMs: performance.now() - perLayerInputsTraceStart,
|
|
2901
|
+
materialized: Array.isArray(perLayerInputs),
|
|
2902
|
+
});
|
|
2903
|
+
}
|
|
1889
2904
|
} catch (error) {
|
|
1890
2905
|
if (isGpuBufferInstance(hiddenStates?.buffer)) {
|
|
1891
2906
|
releaseBuffer(hiddenStates.buffer);
|
|
@@ -1934,7 +2949,7 @@ export class PipelineGenerator {
|
|
|
1934
2949
|
if (this.#state.finitenessBuffer) {
|
|
1935
2950
|
const device = getDevice();
|
|
1936
2951
|
if (device) {
|
|
1937
|
-
device.queue.writeBuffer(this.#state.finitenessBuffer, 0,
|
|
2952
|
+
device.queue.writeBuffer(this.#state.finitenessBuffer, 0, FINITENESS_RESET_WORDS);
|
|
1938
2953
|
}
|
|
1939
2954
|
}
|
|
1940
2955
|
|
|
@@ -1951,6 +2966,8 @@ export class PipelineGenerator {
|
|
|
1951
2966
|
let currentHiddenBuffer = hiddenStates.buffer;
|
|
1952
2967
|
let prefillRecordMs = 0;
|
|
1953
2968
|
let prefillSubmitWaitMs = 0;
|
|
2969
|
+
const layerLoopTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
2970
|
+
const traceLayerHealthEnabled = isTraceEnabled('logits');
|
|
1954
2971
|
try {
|
|
1955
2972
|
for (let l = 0; l < config.numLayers; l++) {
|
|
1956
2973
|
// Per-layer hard cancellation: when the caller's AbortSignal aborts,
|
|
@@ -1987,6 +3004,7 @@ export class PipelineGenerator {
|
|
|
1987
3004
|
|
|
1988
3005
|
const isCheckpoint = useCheckpoints && opts.debugLayers?.includes(l);
|
|
1989
3006
|
const isChunkBoundary = !isCheckpoint
|
|
3007
|
+
&& !traceLayerHealthEnabled
|
|
1990
3008
|
&& currentRecorder
|
|
1991
3009
|
&& l < config.numLayers - 1
|
|
1992
3010
|
&& (l + 1) % prefillRecorderChunkLayers === 0;
|
|
@@ -2002,6 +3020,27 @@ export class PipelineGenerator {
|
|
|
2002
3020
|
currentRecorder = undefined;
|
|
2003
3021
|
}
|
|
2004
3022
|
|
|
3023
|
+
if (traceLayerHealthEnabled && currentRecorder) {
|
|
3024
|
+
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3025
|
+
currentHiddenBuffer,
|
|
3026
|
+
currentRecorder,
|
|
3027
|
+
'prefill_trace_layer_health_carry'
|
|
3028
|
+
);
|
|
3029
|
+
const traceSubmitStart = performance.now();
|
|
3030
|
+
await currentRecorder.submitAndWait();
|
|
3031
|
+
await recordProfile(currentRecorder);
|
|
3032
|
+
prefillSubmitWaitMs += performance.now() - traceSubmitStart;
|
|
3033
|
+
await traceActivationHealth(
|
|
3034
|
+
`PREFILL_LAYER_${l}_HEALTH`,
|
|
3035
|
+
currentHiddenBuffer,
|
|
3036
|
+
activationDtype,
|
|
3037
|
+
numTokens * config.hiddenSize
|
|
3038
|
+
);
|
|
3039
|
+
currentRecorder = l < config.numLayers - 1
|
|
3040
|
+
? createRecorder('prefill-trace')
|
|
3041
|
+
: undefined;
|
|
3042
|
+
}
|
|
3043
|
+
|
|
2005
3044
|
const shouldDebug = opts.debug && currentHiddenBuffer && (!recorder || isCheckpoint);
|
|
2006
3045
|
if (shouldDebug && !currentRecorder) {
|
|
2007
3046
|
const device = getDevice();
|
|
@@ -2056,10 +3095,10 @@ export class PipelineGenerator {
|
|
|
2056
3095
|
// tracked buffers when GPU work completes. Profile runs keep the
|
|
2057
3096
|
// sync path because resolveProfileTimings requires the submitted
|
|
2058
3097
|
// work to be done.
|
|
2059
|
-
const
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
3098
|
+
const chunkSubmitMode = resolvePrefillChunkSubmitMode(
|
|
3099
|
+
this.#state.runtimeConfig,
|
|
3100
|
+
this.#state.modelConfig
|
|
3101
|
+
);
|
|
2063
3102
|
const chunkSubmitStart = performance.now();
|
|
2064
3103
|
if (chunkSubmitMode === 'async' && !opts.profile) {
|
|
2065
3104
|
currentRecorder.submit({ cleanup: 'deferred' });
|
|
@@ -2068,9 +3107,31 @@ export class PipelineGenerator {
|
|
|
2068
3107
|
await recordProfile(currentRecorder);
|
|
2069
3108
|
}
|
|
2070
3109
|
prefillSubmitWaitMs += performance.now() - chunkSubmitStart;
|
|
3110
|
+
await traceActivationHealth(
|
|
3111
|
+
`PREFILL_LAYER_${l}_HEALTH`,
|
|
3112
|
+
currentHiddenBuffer,
|
|
3113
|
+
activationDtype,
|
|
3114
|
+
numTokens * config.hiddenSize
|
|
3115
|
+
);
|
|
3116
|
+
if (tracePrefillEnabled) {
|
|
3117
|
+
trace.perf('Prefill chunk submitted', {
|
|
3118
|
+
layer: l,
|
|
3119
|
+
elapsedMs: performance.now() - layerLoopTraceStart,
|
|
3120
|
+
prefillRecordMs,
|
|
3121
|
+
prefillSubmitWaitMs,
|
|
3122
|
+
});
|
|
3123
|
+
}
|
|
2071
3124
|
currentRecorder = createRecorder('prefill-chunk');
|
|
2072
3125
|
}
|
|
2073
3126
|
}
|
|
3127
|
+
if (tracePrefillEnabled) {
|
|
3128
|
+
trace.perf('Prefill layer loop recorded', {
|
|
3129
|
+
numTokens,
|
|
3130
|
+
elapsedMs: performance.now() - layerLoopTraceStart,
|
|
3131
|
+
prefillRecordMs,
|
|
3132
|
+
prefillSubmitWaitMs,
|
|
3133
|
+
});
|
|
3134
|
+
}
|
|
2074
3135
|
} finally {
|
|
2075
3136
|
context.perLayerInputBuffer = null;
|
|
2076
3137
|
if (perLayerInputs) {
|
|
@@ -2182,6 +3243,7 @@ export class PipelineGenerator {
|
|
|
2182
3243
|
&& !this.#state.disableRecordedLogits
|
|
2183
3244
|
&& numTokens === 1;
|
|
2184
3245
|
if (currentRecorder && canRecordLogits) {
|
|
3246
|
+
const logitsTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
2185
3247
|
const recorded = await recordLogitsGPU(
|
|
2186
3248
|
currentRecorder,
|
|
2187
3249
|
currentHiddenBuffer,
|
|
@@ -2202,6 +3264,13 @@ export class PipelineGenerator {
|
|
|
2202
3264
|
const logitsData = await readBufferSlice(recorded.logitsBuffer, lastLogitsOffset, lastLogitsSize);
|
|
2203
3265
|
releaseBuffer(recorded.logitsBuffer);
|
|
2204
3266
|
lastLogits = decodeReadback(logitsData, recorded.logitsDtype);
|
|
3267
|
+
if (tracePrefillEnabled) {
|
|
3268
|
+
trace.perf('Prefill recorded logits complete', {
|
|
3269
|
+
numTokens,
|
|
3270
|
+
vocabSize: logitsVocabSize,
|
|
3271
|
+
elapsedMs: performance.now() - logitsTraceStart,
|
|
3272
|
+
});
|
|
3273
|
+
}
|
|
2205
3274
|
|
|
2206
3275
|
const health = getLogitsHealth(lastLogits);
|
|
2207
3276
|
if (health.nanCount > 0 || health.infCount > 0 || health.nonZeroCount === 0) {
|
|
@@ -2240,6 +3309,7 @@ export class PipelineGenerator {
|
|
|
2240
3309
|
|
|
2241
3310
|
releaseBuffer(currentHiddenBuffer);
|
|
2242
3311
|
} else {
|
|
3312
|
+
const logitsTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
2243
3313
|
if (currentRecorder) {
|
|
2244
3314
|
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
2245
3315
|
currentHiddenBuffer,
|
|
@@ -2267,6 +3337,13 @@ export class PipelineGenerator {
|
|
|
2267
3337
|
? logits
|
|
2268
3338
|
: extractLastPositionLogits(logits, numTokens, logitsVocabSize);
|
|
2269
3339
|
releaseBuffer(currentHiddenBuffer);
|
|
3340
|
+
if (tracePrefillEnabled) {
|
|
3341
|
+
trace.perf('Prefill logits complete', {
|
|
3342
|
+
numTokens,
|
|
3343
|
+
vocabSize: logitsVocabSize,
|
|
3344
|
+
elapsedMs: performance.now() - logitsTraceStart,
|
|
3345
|
+
});
|
|
3346
|
+
}
|
|
2270
3347
|
}
|
|
2271
3348
|
|
|
2272
3349
|
this.#state.currentSeqLen = startPos + numTokens;
|
|
@@ -2286,6 +3363,9 @@ export class PipelineGenerator {
|
|
|
2286
3363
|
if (opts.debug) {
|
|
2287
3364
|
logitsSanity(lastLogits, 'Prefill', (tokens) => resolveTokenText(this.#state.tokenizer, tokens));
|
|
2288
3365
|
}
|
|
3366
|
+
if (isTraceEnabled('logits')) {
|
|
3367
|
+
trace.logits('PREFILL_LOGITS_HEALTH', getLogitsHealth(lastLogits));
|
|
3368
|
+
}
|
|
2289
3369
|
|
|
2290
3370
|
if (opts.debug) {
|
|
2291
3371
|
if (this.#state.kvCache?.hasGPUCache?.()) {
|
|
@@ -2295,6 +3375,13 @@ export class PipelineGenerator {
|
|
|
2295
3375
|
}
|
|
2296
3376
|
}
|
|
2297
3377
|
|
|
3378
|
+
if (tracePrefillEnabled) {
|
|
3379
|
+
trace.perf('Prefill phase complete', {
|
|
3380
|
+
numTokens,
|
|
3381
|
+
totalMs: performance.now() - prefillTraceStart,
|
|
3382
|
+
});
|
|
3383
|
+
}
|
|
3384
|
+
|
|
2298
3385
|
return lastLogits;
|
|
2299
3386
|
}
|
|
2300
3387
|
|