doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -38,6 +38,8 @@ import {
|
|
|
38
38
|
safeToFixed,
|
|
39
39
|
sampleTimingNumber,
|
|
40
40
|
buildCanonicalTiming,
|
|
41
|
+
buildLoadTimingDiagnostics,
|
|
42
|
+
buildDecodeBottleneckDiagnostics,
|
|
41
43
|
buildTimingDiagnostics,
|
|
42
44
|
} from './browser-harness-suite-helpers.js';
|
|
43
45
|
import {
|
|
@@ -47,6 +49,10 @@ import {
|
|
|
47
49
|
} from './browser-harness-model-helpers.js';
|
|
48
50
|
import {
|
|
49
51
|
resolveBenchmarkRunSettings,
|
|
52
|
+
normalizeDecodeRecordOpLabels,
|
|
53
|
+
normalizeUniformCacheStats,
|
|
54
|
+
buildDecodeRecordTopOps,
|
|
55
|
+
buildDecodeRecordTopOpGroups,
|
|
50
56
|
runEmbeddingSemanticChecks,
|
|
51
57
|
isCoherentOutput,
|
|
52
58
|
runTextInference,
|
|
@@ -64,6 +70,21 @@ import { stableSortObject } from '../utils/stable-sort-object.js';
|
|
|
64
70
|
const TRAINING_SUITE_MODULE_PATH = '../experimental/training/suite.js';
|
|
65
71
|
let trainingSuiteModulePromise = null;
|
|
66
72
|
|
|
73
|
+
function resolvePipelineLoadTimings(pipeline) {
|
|
74
|
+
if (!pipeline || typeof pipeline.getStats !== 'function') {
|
|
75
|
+
return { loadTiming: null, pipelineLoadTiming: null };
|
|
76
|
+
}
|
|
77
|
+
try {
|
|
78
|
+
const stats = pipeline.getStats() ?? {};
|
|
79
|
+
return {
|
|
80
|
+
loadTiming: stats.loadTiming ?? null,
|
|
81
|
+
pipelineLoadTiming: stats.pipelineLoadTiming ?? null,
|
|
82
|
+
};
|
|
83
|
+
} catch {
|
|
84
|
+
return { loadTiming: null, pipelineLoadTiming: null };
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
67
88
|
async function loadTrainingSuiteModule() {
|
|
68
89
|
if (!trainingSuiteModulePromise) {
|
|
69
90
|
trainingSuiteModulePromise = import(TRAINING_SUITE_MODULE_PATH);
|
|
@@ -125,6 +146,83 @@ const BROWSER_WORKLOAD_DISPATCH_MAP = Object.freeze({
|
|
|
125
146
|
}),
|
|
126
147
|
});
|
|
127
148
|
|
|
149
|
+
function isPlainObject(value) {
|
|
150
|
+
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function resolveExecutionPlanCadence(executionPlan) {
|
|
154
|
+
if (!isPlainObject(executionPlan)) {
|
|
155
|
+
return null;
|
|
156
|
+
}
|
|
157
|
+
const finalActivePlanId = typeof executionPlan.finalActivePlanId === 'string'
|
|
158
|
+
? executionPlan.finalActivePlanId
|
|
159
|
+
: null;
|
|
160
|
+
for (const candidate of [executionPlan.primary, executionPlan.fallback]) {
|
|
161
|
+
if (!isPlainObject(candidate)) {
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
if (finalActivePlanId == null || candidate.id === finalActivePlanId) {
|
|
165
|
+
return candidate;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return isPlainObject(executionPlan.primary) ? executionPlan.primary : null;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function resolveDecodeCadence(runtimeConfig, executionPlan = null) {
|
|
172
|
+
const inference = runtimeConfig?.inference;
|
|
173
|
+
const batching = inference?.batching;
|
|
174
|
+
const session = inference?.session;
|
|
175
|
+
const decodeLoop = session?.decodeLoop;
|
|
176
|
+
if (!isPlainObject(batching) || !isPlainObject(decodeLoop)) {
|
|
177
|
+
return null;
|
|
178
|
+
}
|
|
179
|
+
const executionPlanCadence = resolveExecutionPlanCadence(executionPlan);
|
|
180
|
+
const batchSize = executionPlanCadence?.batchSize ?? decodeLoop.batchSize ?? batching.batchSize ?? null;
|
|
181
|
+
const readbackInterval = executionPlanCadence?.readbackInterval ?? decodeLoop.readbackInterval ?? batching.readbackInterval ?? null;
|
|
182
|
+
return {
|
|
183
|
+
batchSize,
|
|
184
|
+
readbackInterval,
|
|
185
|
+
stopCheckMode: executionPlanCadence?.stopCheckMode ?? decodeLoop.stopCheckMode ?? batching.stopCheckMode ?? null,
|
|
186
|
+
readbackMode: executionPlanCadence?.readbackMode ?? decodeLoop.readbackMode ?? batching.readbackMode ?? null,
|
|
187
|
+
disableCommandBatching: executionPlanCadence?.disableCommandBatching ?? decodeLoop.disableCommandBatching ?? null,
|
|
188
|
+
disableMultiTokenDecode: inference?.generation?.disableMultiTokenDecode === true,
|
|
189
|
+
speculationMode: session?.speculation?.mode ?? null,
|
|
190
|
+
tokensPerReadback: Number.isFinite(batchSize) && Number.isFinite(readbackInterval)
|
|
191
|
+
? batchSize * readbackInterval
|
|
192
|
+
: null,
|
|
193
|
+
runtimeMirror: {
|
|
194
|
+
batching: {
|
|
195
|
+
batchSize: batching.batchSize ?? null,
|
|
196
|
+
readbackInterval: batching.readbackInterval ?? null,
|
|
197
|
+
stopCheckMode: batching.stopCheckMode ?? null,
|
|
198
|
+
readbackMode: batching.readbackMode ?? null,
|
|
199
|
+
},
|
|
200
|
+
decodeLoop: {
|
|
201
|
+
batchSize: decodeLoop.batchSize ?? null,
|
|
202
|
+
readbackInterval: decodeLoop.readbackInterval ?? null,
|
|
203
|
+
stopCheckMode: decodeLoop.stopCheckMode ?? null,
|
|
204
|
+
readbackMode: decodeLoop.readbackMode ?? null,
|
|
205
|
+
ringTokens: decodeLoop.ringTokens ?? null,
|
|
206
|
+
ringStop: decodeLoop.ringStop ?? null,
|
|
207
|
+
ringStaging: decodeLoop.ringStaging ?? null,
|
|
208
|
+
},
|
|
209
|
+
},
|
|
210
|
+
executionPlan: executionPlanCadence
|
|
211
|
+
? {
|
|
212
|
+
id: executionPlanCadence.id ?? null,
|
|
213
|
+
batchSize: executionPlanCadence.batchSize ?? null,
|
|
214
|
+
readbackInterval: executionPlanCadence.readbackInterval ?? null,
|
|
215
|
+
stopCheckMode: executionPlanCadence.stopCheckMode ?? null,
|
|
216
|
+
readbackMode: executionPlanCadence.readbackMode ?? null,
|
|
217
|
+
disableCommandBatching: executionPlanCadence.disableCommandBatching ?? null,
|
|
218
|
+
ringTokens: executionPlanCadence.ringTokens ?? null,
|
|
219
|
+
ringStop: executionPlanCadence.ringStop ?? null,
|
|
220
|
+
ringStaging: executionPlanCadence.ringStaging ?? null,
|
|
221
|
+
}
|
|
222
|
+
: null,
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
|
|
128
226
|
export function getBrowserSupportedSuites() {
|
|
129
227
|
return [...BROWSER_WORKLOAD_SET];
|
|
130
228
|
}
|
|
@@ -566,6 +664,7 @@ async function runInferenceSuite(options = {}) {
|
|
|
566
664
|
firstResponseMs: safeToFixed(safeModelLoadMs + run.phase.ttftMs),
|
|
567
665
|
prefillMs: safeToFixed(run.phase.prefillMs),
|
|
568
666
|
decodeMs: safeToFixed(run.phase.decodeMs),
|
|
667
|
+
wallRunMs: safeToFixed(run.phase.wallMs ?? run.durationMs),
|
|
569
668
|
prefillTokens: Math.round(run.phase.prefillTokens),
|
|
570
669
|
decodeTokens: Math.round(run.phase.decodeTokens),
|
|
571
670
|
stopReason: run.phase.stopReason ?? null,
|
|
@@ -575,6 +674,7 @@ async function runInferenceSuite(options = {}) {
|
|
|
575
674
|
decodeTokensPerSec: safeToFixed(run.phase.decodeTokensPerSec),
|
|
576
675
|
modelLoadMs: safeModelLoadMs,
|
|
577
676
|
gpu: run.phase.gpu,
|
|
677
|
+
batching: run.phase.batching ?? null,
|
|
578
678
|
plePreparedTokenCache: run.phase.plePreparedTokenCache ?? null,
|
|
579
679
|
prefillProfileSteps: run.phase.prefillProfileSteps,
|
|
580
680
|
decodeProfileSteps: run.phase.decodeProfileSteps,
|
|
@@ -594,6 +694,12 @@ async function runInferenceSuite(options = {}) {
|
|
|
594
694
|
const memoryStats = typeof harness.pipeline?.getMemoryStats === 'function'
|
|
595
695
|
? harness.pipeline.getMemoryStats()
|
|
596
696
|
: null;
|
|
697
|
+
const loadTimings = resolvePipelineLoadTimings(harness.pipeline);
|
|
698
|
+
const loadDiagnostics = buildLoadTimingDiagnostics(
|
|
699
|
+
safeModelLoadMs,
|
|
700
|
+
loadTimings.loadTiming,
|
|
701
|
+
loadTimings.pipelineLoadTiming
|
|
702
|
+
);
|
|
597
703
|
if (typeof harness.pipeline.unload === 'function' && !options.keepPipeline) {
|
|
598
704
|
await harness.pipeline.unload();
|
|
599
705
|
}
|
|
@@ -619,7 +725,16 @@ async function runInferenceSuite(options = {}) {
|
|
|
619
725
|
const timingDiagnostics = buildTimingDiagnostics(timing, {
|
|
620
726
|
source: 'doppler',
|
|
621
727
|
prefillSemantics: 'internal_prefill_phase',
|
|
728
|
+
loadTiming: loadTimings.loadTiming,
|
|
729
|
+
pipelineLoadTiming: loadTimings.pipelineLoadTiming,
|
|
622
730
|
});
|
|
731
|
+
const decodeBottleneck = buildDecodeBottleneckDiagnostics(metrics, timing);
|
|
732
|
+
const metricsWithTimingDiagnostics = decodeBottleneck
|
|
733
|
+
? { ...metrics, decodeBottleneck }
|
|
734
|
+
: metrics;
|
|
735
|
+
if (decodeBottleneck) {
|
|
736
|
+
timingDiagnostics.decodeBottleneck = decodeBottleneck;
|
|
737
|
+
}
|
|
623
738
|
const firstLoad = buildFirstLoadComposition({
|
|
624
739
|
modelLoadMs: timing.modelLoadMs,
|
|
625
740
|
firstTokenMs: timing.firstTokenMs,
|
|
@@ -627,7 +742,9 @@ async function runInferenceSuite(options = {}) {
|
|
|
627
742
|
});
|
|
628
743
|
const metricsWithContracts = buildSuiteContractMetrics(
|
|
629
744
|
options.suiteName || 'inference',
|
|
630
|
-
|
|
745
|
+
loadDiagnostics
|
|
746
|
+
? { ...metricsWithTimingDiagnostics, load: loadDiagnostics }
|
|
747
|
+
: metricsWithTimingDiagnostics,
|
|
631
748
|
harness.manifest
|
|
632
749
|
);
|
|
633
750
|
return {
|
|
@@ -730,7 +847,7 @@ async function runBenchSuite(options = {}) {
|
|
|
730
847
|
};
|
|
731
848
|
}
|
|
732
849
|
|
|
733
|
-
if (workloadType === 'diffusion') {
|
|
850
|
+
if (workloadType === 'diffusion' || workloadType === 'diffusion_gemma') {
|
|
734
851
|
const diffusionBench = await runDiffusionSuite({
|
|
735
852
|
...options,
|
|
736
853
|
command: 'bench',
|
|
@@ -757,7 +874,7 @@ async function runBenchSuite(options = {}) {
|
|
|
757
874
|
results: benchResults,
|
|
758
875
|
metrics: {
|
|
759
876
|
...(diffusionBench.metrics || {}),
|
|
760
|
-
workloadType
|
|
877
|
+
workloadType,
|
|
761
878
|
},
|
|
762
879
|
};
|
|
763
880
|
}
|
|
@@ -893,6 +1010,7 @@ async function runBenchSuite(options = {}) {
|
|
|
893
1010
|
} else {
|
|
894
1011
|
const tokensPerSec = [];
|
|
895
1012
|
const durations = [];
|
|
1013
|
+
const phaseTotals = [];
|
|
896
1014
|
const tokensGenerated = [];
|
|
897
1015
|
const decodeMsPerToken = [];
|
|
898
1016
|
const ttftMs = [];
|
|
@@ -906,19 +1024,41 @@ async function runBenchSuite(options = {}) {
|
|
|
906
1024
|
const gpuPrefillMs = [];
|
|
907
1025
|
const gpuDecodeMs = [];
|
|
908
1026
|
const gpuDecodeRecordMs = [];
|
|
1027
|
+
const gpuDecodeRecordOps = [];
|
|
1028
|
+
const gpuDecodeRecordPasses = [];
|
|
1029
|
+
const gpuDecodeRecordMsPerOp = [];
|
|
1030
|
+
const gpuDecodeRecordMsPerPass = [];
|
|
1031
|
+
const gpuDecodeRecordPassesPerOp = [];
|
|
1032
|
+
const gpuDecodeRecordMsPerExecutedBatchToken = [];
|
|
1033
|
+
const gpuDecodeRecordOpsPerExecutedBatchToken = [];
|
|
1034
|
+
const gpuDecodeRecordPassesPerExecutedBatchToken = [];
|
|
1035
|
+
const gpuDecodeRecordOpLabels = {};
|
|
1036
|
+
let hasGpuDecodeRecordOpLabels = false;
|
|
909
1037
|
const gpuDecodeSubmitWaitMs = [];
|
|
910
1038
|
const gpuDecodeReadbackWaitMs = [];
|
|
1039
|
+
const gpuDecodeReadbackMapWaitMs = [];
|
|
1040
|
+
const gpuDecodeReadbackCleanupMs = [];
|
|
1041
|
+
const gpuDecodeReadbackCopyMs = [];
|
|
911
1042
|
const gpuDecodeOrchestrationMs = [];
|
|
912
1043
|
const gpuPrefillRecordMs = [];
|
|
913
1044
|
const gpuPrefillSubmitWaitMs = [];
|
|
914
1045
|
const singleTokenSubmitWaitMs = [];
|
|
915
1046
|
const singleTokenReadbackWaitMs = [];
|
|
1047
|
+
const singleTokenReadbackMapWaitMs = [];
|
|
1048
|
+
const singleTokenReadbackCleanupMs = [];
|
|
1049
|
+
const singleTokenReadbackCopyMs = [];
|
|
916
1050
|
const singleTokenOrchestrationMs = [];
|
|
917
1051
|
const batchedForwardCalls = [];
|
|
918
1052
|
const unbatchedForwardCalls = [];
|
|
919
1053
|
const totalBatchedTimeMs = [];
|
|
920
1054
|
const totalUnbatchedTimeMs = [];
|
|
921
1055
|
const gpuSubmissions = [];
|
|
1056
|
+
const requestedBatchTokens = [];
|
|
1057
|
+
const effectiveBatchTokens = [];
|
|
1058
|
+
const executedBatchTokens = [];
|
|
1059
|
+
const resolvedBatchTokens = [];
|
|
1060
|
+
const maxBatchTokenCap = [];
|
|
1061
|
+
const batchClampCount = [];
|
|
922
1062
|
const plePreparedTokenCacheHits = [];
|
|
923
1063
|
const plePreparedTokenCacheMisses = [];
|
|
924
1064
|
const plePreparedTokenCacheEntries = [];
|
|
@@ -926,8 +1066,13 @@ async function runBenchSuite(options = {}) {
|
|
|
926
1066
|
|
|
927
1067
|
let generatedText = null;
|
|
928
1068
|
let generatedPromptInput = null;
|
|
1069
|
+
let generatedReferenceTranscript = null;
|
|
1070
|
+
let lastPromptLabel = benchRun.promptLabel;
|
|
1071
|
+
let lastMaxTokens = benchRun.maxTokens;
|
|
929
1072
|
let lastDecodeMode = null;
|
|
930
1073
|
let lastBatchGuardReason = null;
|
|
1074
|
+
let lastExecutionPlan = null;
|
|
1075
|
+
let lastGpuUniformCache = null;
|
|
931
1076
|
for (let i = 0; i < warmupRuns + timedRuns; i++) {
|
|
932
1077
|
harness.pipeline.reset?.();
|
|
933
1078
|
const run = await withHarnessPhase(
|
|
@@ -947,8 +1092,15 @@ async function runBenchSuite(options = {}) {
|
|
|
947
1092
|
if (i === warmupRuns + timedRuns - 1) {
|
|
948
1093
|
generatedText = run?.output ?? null;
|
|
949
1094
|
generatedPromptInput = run?.promptInput ?? null;
|
|
1095
|
+
generatedReferenceTranscript = buildReferenceTranscriptSeed(run, {
|
|
1096
|
+
executionGraphHash: resolveExecutionGraphHash(harness.manifest),
|
|
1097
|
+
kvCache: run?.phase?.kvCache ?? null,
|
|
1098
|
+
});
|
|
1099
|
+
lastPromptLabel = run?.prompt ?? benchRun.promptLabel;
|
|
1100
|
+
lastMaxTokens = Number.isFinite(run?.maxTokens) ? run.maxTokens : benchRun.maxTokens;
|
|
950
1101
|
lastDecodeMode = run?.phase?.decodeMode ?? null;
|
|
951
1102
|
lastBatchGuardReason = run?.phase?.batchGuardReason ?? null;
|
|
1103
|
+
lastExecutionPlan = run?.phase?.executionPlan ?? null;
|
|
952
1104
|
}
|
|
953
1105
|
if (i >= warmupRuns) {
|
|
954
1106
|
const phase = run?.phase ?? {};
|
|
@@ -958,6 +1110,7 @@ async function runBenchSuite(options = {}) {
|
|
|
958
1110
|
const phasePlePreparedTokenCache = phase.plePreparedTokenCache;
|
|
959
1111
|
tokensPerSec.push(run?.tokensPerSec);
|
|
960
1112
|
durations.push(run?.durationMs);
|
|
1113
|
+
phaseTotals.push(phase.totalMs);
|
|
961
1114
|
tokensGenerated.push(phaseTokens.length);
|
|
962
1115
|
ttftMs.push(phase.ttftMs);
|
|
963
1116
|
prefillMs.push(phase.prefillMs);
|
|
@@ -970,11 +1123,45 @@ async function runBenchSuite(options = {}) {
|
|
|
970
1123
|
if (phase.decodeMs > 0 && phase.decodeTokens > 0) {
|
|
971
1124
|
decodeMsPerToken.push(phase.decodeMs / phase.decodeTokens);
|
|
972
1125
|
}
|
|
1126
|
+
const phaseGpuUniformCache = normalizeUniformCacheStats(phaseGpu?.uniformCache);
|
|
1127
|
+
if (phaseGpuUniformCache) {
|
|
1128
|
+
lastGpuUniformCache = phaseGpuUniformCache;
|
|
1129
|
+
}
|
|
973
1130
|
if (Number.isFinite(phaseGpu?.prefillMs)) gpuPrefillMs.push(phaseGpu.prefillMs);
|
|
974
1131
|
if (Number.isFinite(phaseGpu?.decodeMs)) gpuDecodeMs.push(phaseGpu.decodeMs);
|
|
975
1132
|
if (Number.isFinite(phaseGpu?.decodeRecordMs)) gpuDecodeRecordMs.push(phaseGpu.decodeRecordMs);
|
|
1133
|
+
if (Number.isFinite(phaseGpu?.decodeRecordOps)) gpuDecodeRecordOps.push(phaseGpu.decodeRecordOps);
|
|
1134
|
+
if (Number.isFinite(phaseGpu?.decodeRecordPasses)) gpuDecodeRecordPasses.push(phaseGpu.decodeRecordPasses);
|
|
1135
|
+
const phaseDecodeRecordOpLabels = normalizeDecodeRecordOpLabels(phaseGpu?.decodeRecordOpLabels);
|
|
1136
|
+
if (phaseDecodeRecordOpLabels) {
|
|
1137
|
+
hasGpuDecodeRecordOpLabels = true;
|
|
1138
|
+
for (const [label, count] of Object.entries(phaseDecodeRecordOpLabels)) {
|
|
1139
|
+
gpuDecodeRecordOpLabels[label] = (gpuDecodeRecordOpLabels[label] ?? 0) + count;
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
if (Number.isFinite(phaseGpu?.decodeRecordMsPerOp)) {
|
|
1143
|
+
gpuDecodeRecordMsPerOp.push(phaseGpu.decodeRecordMsPerOp);
|
|
1144
|
+
}
|
|
1145
|
+
if (Number.isFinite(phaseGpu?.decodeRecordMsPerPass)) {
|
|
1146
|
+
gpuDecodeRecordMsPerPass.push(phaseGpu.decodeRecordMsPerPass);
|
|
1147
|
+
}
|
|
1148
|
+
if (Number.isFinite(phaseGpu?.decodeRecordPassesPerOp)) {
|
|
1149
|
+
gpuDecodeRecordPassesPerOp.push(phaseGpu.decodeRecordPassesPerOp);
|
|
1150
|
+
}
|
|
1151
|
+
if (Number.isFinite(phaseGpu?.decodeRecordMsPerExecutedBatchToken)) {
|
|
1152
|
+
gpuDecodeRecordMsPerExecutedBatchToken.push(phaseGpu.decodeRecordMsPerExecutedBatchToken);
|
|
1153
|
+
}
|
|
1154
|
+
if (Number.isFinite(phaseGpu?.decodeRecordOpsPerExecutedBatchToken)) {
|
|
1155
|
+
gpuDecodeRecordOpsPerExecutedBatchToken.push(phaseGpu.decodeRecordOpsPerExecutedBatchToken);
|
|
1156
|
+
}
|
|
1157
|
+
if (Number.isFinite(phaseGpu?.decodeRecordPassesPerExecutedBatchToken)) {
|
|
1158
|
+
gpuDecodeRecordPassesPerExecutedBatchToken.push(phaseGpu.decodeRecordPassesPerExecutedBatchToken);
|
|
1159
|
+
}
|
|
976
1160
|
if (Number.isFinite(phaseGpu?.decodeSubmitWaitMs)) gpuDecodeSubmitWaitMs.push(phaseGpu.decodeSubmitWaitMs);
|
|
977
1161
|
if (Number.isFinite(phaseGpu?.decodeReadbackWaitMs)) gpuDecodeReadbackWaitMs.push(phaseGpu.decodeReadbackWaitMs);
|
|
1162
|
+
if (Number.isFinite(phaseGpu?.decodeReadbackMapWaitMs)) gpuDecodeReadbackMapWaitMs.push(phaseGpu.decodeReadbackMapWaitMs);
|
|
1163
|
+
if (Number.isFinite(phaseGpu?.decodeReadbackCleanupMs)) gpuDecodeReadbackCleanupMs.push(phaseGpu.decodeReadbackCleanupMs);
|
|
1164
|
+
if (Number.isFinite(phaseGpu?.decodeReadbackCopyMs)) gpuDecodeReadbackCopyMs.push(phaseGpu.decodeReadbackCopyMs);
|
|
978
1165
|
if (Number.isFinite(phaseGpu?.prefillRecordMs)) gpuPrefillRecordMs.push(phaseGpu.prefillRecordMs);
|
|
979
1166
|
if (Number.isFinite(phaseGpu?.prefillSubmitWaitMs)) gpuPrefillSubmitWaitMs.push(phaseGpu.prefillSubmitWaitMs);
|
|
980
1167
|
if (Number.isFinite(phaseGpu?.decodeOrchestrationMs)) {
|
|
@@ -982,12 +1169,21 @@ async function runBenchSuite(options = {}) {
|
|
|
982
1169
|
}
|
|
983
1170
|
if (Number.isFinite(phaseGpu?.singleTokenSubmitWaitMs)) singleTokenSubmitWaitMs.push(phaseGpu.singleTokenSubmitWaitMs);
|
|
984
1171
|
if (Number.isFinite(phaseGpu?.singleTokenReadbackWaitMs)) singleTokenReadbackWaitMs.push(phaseGpu.singleTokenReadbackWaitMs);
|
|
1172
|
+
if (Number.isFinite(phaseGpu?.singleTokenReadbackMapWaitMs)) singleTokenReadbackMapWaitMs.push(phaseGpu.singleTokenReadbackMapWaitMs);
|
|
1173
|
+
if (Number.isFinite(phaseGpu?.singleTokenReadbackCleanupMs)) singleTokenReadbackCleanupMs.push(phaseGpu.singleTokenReadbackCleanupMs);
|
|
1174
|
+
if (Number.isFinite(phaseGpu?.singleTokenReadbackCopyMs)) singleTokenReadbackCopyMs.push(phaseGpu.singleTokenReadbackCopyMs);
|
|
985
1175
|
if (Number.isFinite(phaseGpu?.singleTokenOrchestrationMs)) singleTokenOrchestrationMs.push(phaseGpu.singleTokenOrchestrationMs);
|
|
986
1176
|
if (Number.isFinite(phaseBatching?.batchedForwardCalls)) batchedForwardCalls.push(phaseBatching.batchedForwardCalls);
|
|
987
1177
|
if (Number.isFinite(phaseBatching?.unbatchedForwardCalls)) unbatchedForwardCalls.push(phaseBatching.unbatchedForwardCalls);
|
|
988
1178
|
if (Number.isFinite(phaseBatching?.totalBatchedTimeMs)) totalBatchedTimeMs.push(phaseBatching.totalBatchedTimeMs);
|
|
989
1179
|
if (Number.isFinite(phaseBatching?.totalUnbatchedTimeMs)) totalUnbatchedTimeMs.push(phaseBatching.totalUnbatchedTimeMs);
|
|
990
1180
|
if (Number.isFinite(phaseBatching?.gpuSubmissions)) gpuSubmissions.push(phaseBatching.gpuSubmissions);
|
|
1181
|
+
if (Number.isFinite(phaseBatching?.requestedBatchTokens)) requestedBatchTokens.push(phaseBatching.requestedBatchTokens);
|
|
1182
|
+
if (Number.isFinite(phaseBatching?.effectiveBatchTokens)) effectiveBatchTokens.push(phaseBatching.effectiveBatchTokens);
|
|
1183
|
+
if (Number.isFinite(phaseBatching?.executedBatchTokens)) executedBatchTokens.push(phaseBatching.executedBatchTokens);
|
|
1184
|
+
if (Number.isFinite(phaseBatching?.resolvedBatchTokens)) resolvedBatchTokens.push(phaseBatching.resolvedBatchTokens);
|
|
1185
|
+
if (Number.isFinite(phaseBatching?.maxBatchTokenCap)) maxBatchTokenCap.push(phaseBatching.maxBatchTokenCap);
|
|
1186
|
+
if (Number.isFinite(phaseBatching?.batchClampCount)) batchClampCount.push(phaseBatching.batchClampCount);
|
|
991
1187
|
if (Number.isFinite(phasePlePreparedTokenCache?.hits)) plePreparedTokenCacheHits.push(phasePlePreparedTokenCache.hits);
|
|
992
1188
|
if (Number.isFinite(phasePlePreparedTokenCache?.misses)) plePreparedTokenCacheMisses.push(phasePlePreparedTokenCache.misses);
|
|
993
1189
|
if (Number.isFinite(phasePlePreparedTokenCache?.entries)) plePreparedTokenCacheEntries.push(phasePlePreparedTokenCache.entries);
|
|
@@ -995,7 +1191,8 @@ async function runBenchSuite(options = {}) {
|
|
|
995
1191
|
}
|
|
996
1192
|
}
|
|
997
1193
|
|
|
998
|
-
const totalMsStats = computeSampleStats(
|
|
1194
|
+
const totalMsStats = computeSampleStats(phaseTotals);
|
|
1195
|
+
const wallRunMsStats = computeSampleStats(durations);
|
|
999
1196
|
const tokensPerSecStats = computeSampleStats(tokensPerSec);
|
|
1000
1197
|
const decodeTokensPerSecStats = computeSampleStats(decodeTokensPerSec);
|
|
1001
1198
|
const prefillTokensPerSecStats = computeSampleStats(prefillTokensPerSec);
|
|
@@ -1007,23 +1204,74 @@ async function runBenchSuite(options = {}) {
|
|
|
1007
1204
|
const tokensGeneratedStats = computeSampleStats(tokensGenerated);
|
|
1008
1205
|
const prefillTokensStats = computeSampleStats(prefillTokens);
|
|
1009
1206
|
const decodeTokensStats = computeSampleStats(decodeTokens);
|
|
1207
|
+
const gpuDecodeRecordOpsStats = computeSampleStats(gpuDecodeRecordOps);
|
|
1208
|
+
const gpuDecodeRecordPassesStats = computeSampleStats(gpuDecodeRecordPasses);
|
|
1209
|
+
const gpuDecodeRecordOpLabelSampleCount = gpuDecodeRecordOps.length > 0
|
|
1210
|
+
? gpuDecodeRecordOps.length
|
|
1211
|
+
: 1;
|
|
1212
|
+
const gpuDecodeRecordMeanOpLabels = {};
|
|
1213
|
+
if (hasGpuDecodeRecordOpLabels) {
|
|
1214
|
+
for (const [label, count] of Object.entries(gpuDecodeRecordOpLabels)) {
|
|
1215
|
+
gpuDecodeRecordMeanOpLabels[label] = count / gpuDecodeRecordOpLabelSampleCount;
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1010
1218
|
const hasGpuStats = gpuPrefillMs.length > 0 || gpuDecodeMs.length > 0 || gpuDecodeRecordMs.length > 0
|
|
1219
|
+
|| gpuDecodeRecordOps.length > 0 || gpuDecodeRecordPasses.length > 0
|
|
1220
|
+
|| gpuDecodeRecordMsPerOp.length > 0 || gpuDecodeRecordMsPerPass.length > 0
|
|
1221
|
+
|| gpuDecodeRecordPassesPerOp.length > 0
|
|
1222
|
+
|| gpuDecodeRecordMsPerExecutedBatchToken.length > 0
|
|
1223
|
+
|| gpuDecodeRecordOpsPerExecutedBatchToken.length > 0
|
|
1224
|
+
|| gpuDecodeRecordPassesPerExecutedBatchToken.length > 0
|
|
1225
|
+
|| hasGpuDecodeRecordOpLabels
|
|
1011
1226
|
|| gpuDecodeSubmitWaitMs.length > 0 || gpuDecodeReadbackWaitMs.length > 0
|
|
1227
|
+
|| gpuDecodeReadbackMapWaitMs.length > 0 || gpuDecodeReadbackCleanupMs.length > 0
|
|
1228
|
+
|| gpuDecodeReadbackCopyMs.length > 0
|
|
1012
1229
|
|| gpuDecodeOrchestrationMs.length > 0
|
|
1230
|
+
|| lastGpuUniformCache
|
|
1013
1231
|
|| singleTokenSubmitWaitMs.length > 0 || singleTokenReadbackWaitMs.length > 0
|
|
1232
|
+
|| singleTokenReadbackMapWaitMs.length > 0 || singleTokenReadbackCleanupMs.length > 0
|
|
1233
|
+
|| singleTokenReadbackCopyMs.length > 0
|
|
1014
1234
|
|| singleTokenOrchestrationMs.length > 0;
|
|
1015
1235
|
const gpuPhaseStats = hasGpuStats
|
|
1016
1236
|
? {
|
|
1017
1237
|
prefillMs: computeSampleStats(gpuPrefillMs),
|
|
1018
1238
|
decodeMs: computeSampleStats(gpuDecodeMs),
|
|
1019
1239
|
decodeRecordMs: computeSampleStats(gpuDecodeRecordMs),
|
|
1240
|
+
decodeRecordOps: gpuDecodeRecordOpsStats,
|
|
1241
|
+
decodeRecordPasses: gpuDecodeRecordPassesStats,
|
|
1242
|
+
decodeRecordMsPerOp: computeSampleStats(gpuDecodeRecordMsPerOp),
|
|
1243
|
+
decodeRecordMsPerPass: computeSampleStats(gpuDecodeRecordMsPerPass),
|
|
1244
|
+
decodeRecordPassesPerOp: computeSampleStats(gpuDecodeRecordPassesPerOp),
|
|
1245
|
+
decodeRecordMsPerExecutedBatchToken: computeSampleStats(gpuDecodeRecordMsPerExecutedBatchToken),
|
|
1246
|
+
decodeRecordOpsPerExecutedBatchToken: computeSampleStats(gpuDecodeRecordOpsPerExecutedBatchToken),
|
|
1247
|
+
decodeRecordPassesPerExecutedBatchToken: computeSampleStats(gpuDecodeRecordPassesPerExecutedBatchToken),
|
|
1248
|
+
decodeRecordUniqueOpLabels: hasGpuDecodeRecordOpLabels ? Object.keys(gpuDecodeRecordOpLabels).length : null,
|
|
1249
|
+
decodeRecordTopOps: hasGpuDecodeRecordOpLabels
|
|
1250
|
+
? buildDecodeRecordTopOps(
|
|
1251
|
+
gpuDecodeRecordMeanOpLabels,
|
|
1252
|
+
gpuDecodeRecordOpsStats?.mean
|
|
1253
|
+
)
|
|
1254
|
+
: [],
|
|
1255
|
+
decodeRecordTopOpGroups: hasGpuDecodeRecordOpLabels
|
|
1256
|
+
? buildDecodeRecordTopOpGroups(
|
|
1257
|
+
gpuDecodeRecordMeanOpLabels,
|
|
1258
|
+
gpuDecodeRecordOpsStats?.mean
|
|
1259
|
+
)
|
|
1260
|
+
: [],
|
|
1020
1261
|
decodeSubmitWaitMs: computeSampleStats(gpuDecodeSubmitWaitMs),
|
|
1021
1262
|
decodeReadbackWaitMs: computeSampleStats(gpuDecodeReadbackWaitMs),
|
|
1263
|
+
decodeReadbackMapWaitMs: computeSampleStats(gpuDecodeReadbackMapWaitMs),
|
|
1264
|
+
decodeReadbackCleanupMs: computeSampleStats(gpuDecodeReadbackCleanupMs),
|
|
1265
|
+
decodeReadbackCopyMs: computeSampleStats(gpuDecodeReadbackCopyMs),
|
|
1022
1266
|
decodeOrchestrationMs: computeSampleStats(gpuDecodeOrchestrationMs),
|
|
1023
1267
|
prefillRecordMs: computeSampleStats(gpuPrefillRecordMs),
|
|
1024
1268
|
prefillSubmitWaitMs: computeSampleStats(gpuPrefillSubmitWaitMs),
|
|
1269
|
+
uniformCache: lastGpuUniformCache,
|
|
1025
1270
|
singleTokenSubmitWaitMs: computeSampleStats(singleTokenSubmitWaitMs),
|
|
1026
1271
|
singleTokenReadbackWaitMs: computeSampleStats(singleTokenReadbackWaitMs),
|
|
1272
|
+
singleTokenReadbackMapWaitMs: computeSampleStats(singleTokenReadbackMapWaitMs),
|
|
1273
|
+
singleTokenReadbackCleanupMs: computeSampleStats(singleTokenReadbackCleanupMs),
|
|
1274
|
+
singleTokenReadbackCopyMs: computeSampleStats(singleTokenReadbackCopyMs),
|
|
1027
1275
|
singleTokenOrchestrationMs: computeSampleStats(singleTokenOrchestrationMs),
|
|
1028
1276
|
}
|
|
1029
1277
|
: null;
|
|
@@ -1031,7 +1279,13 @@ async function runBenchSuite(options = {}) {
|
|
|
1031
1279
|
|| unbatchedForwardCalls.length > 0
|
|
1032
1280
|
|| totalBatchedTimeMs.length > 0
|
|
1033
1281
|
|| totalUnbatchedTimeMs.length > 0
|
|
1034
|
-
|| gpuSubmissions.length > 0
|
|
1282
|
+
|| gpuSubmissions.length > 0
|
|
1283
|
+
|| requestedBatchTokens.length > 0
|
|
1284
|
+
|| effectiveBatchTokens.length > 0
|
|
1285
|
+
|| executedBatchTokens.length > 0
|
|
1286
|
+
|| resolvedBatchTokens.length > 0
|
|
1287
|
+
|| maxBatchTokenCap.length > 0
|
|
1288
|
+
|| batchClampCount.length > 0;
|
|
1035
1289
|
const batchingPhaseStats = hasBatchingStats
|
|
1036
1290
|
? {
|
|
1037
1291
|
batchedForwardCalls: computeSampleStats(batchedForwardCalls),
|
|
@@ -1039,6 +1293,12 @@ async function runBenchSuite(options = {}) {
|
|
|
1039
1293
|
totalBatchedTimeMs: computeSampleStats(totalBatchedTimeMs),
|
|
1040
1294
|
totalUnbatchedTimeMs: computeSampleStats(totalUnbatchedTimeMs),
|
|
1041
1295
|
gpuSubmissions: computeSampleStats(gpuSubmissions),
|
|
1296
|
+
requestedBatchTokens: computeSampleStats(requestedBatchTokens),
|
|
1297
|
+
effectiveBatchTokens: computeSampleStats(effectiveBatchTokens),
|
|
1298
|
+
executedBatchTokens: computeSampleStats(executedBatchTokens),
|
|
1299
|
+
resolvedBatchTokens: computeSampleStats(resolvedBatchTokens),
|
|
1300
|
+
maxBatchTokenCap: computeSampleStats(maxBatchTokenCap),
|
|
1301
|
+
batchClampCount: computeSampleStats(batchClampCount),
|
|
1042
1302
|
}
|
|
1043
1303
|
: null;
|
|
1044
1304
|
const hasPlePreparedTokenCacheStats = plePreparedTokenCacheHits.length > 0
|
|
@@ -1069,8 +1329,8 @@ async function runBenchSuite(options = {}) {
|
|
|
1069
1329
|
warmupRuns,
|
|
1070
1330
|
timedRuns,
|
|
1071
1331
|
...(Number.isFinite(benchRun?.seed) ? { seed: benchRun.seed } : {}),
|
|
1072
|
-
prompt:
|
|
1073
|
-
maxTokens:
|
|
1332
|
+
prompt: lastPromptLabel,
|
|
1333
|
+
maxTokens: lastMaxTokens,
|
|
1074
1334
|
decodeTokensPerSec: sampleTimingNumber(decodeTokensPerSecStats, 'median'),
|
|
1075
1335
|
avgTokensGenerated: Math.round(tokensGeneratedStats.mean),
|
|
1076
1336
|
avgPrefillTokens: Math.round(prefillTokensStats.mean),
|
|
@@ -1089,6 +1349,7 @@ async function runBenchSuite(options = {}) {
|
|
|
1089
1349
|
decodeMsPerTokenP95: sampleTimingNumber(decodeMsPerTokenStats, 'p95'),
|
|
1090
1350
|
decodeMsPerTokenP99: sampleTimingNumber(decodeMsPerTokenStats, 'p99'),
|
|
1091
1351
|
avgPrefillMs: sampleTimingNumber(prefillMsStats, 'mean'),
|
|
1352
|
+
wallRunMs: sampleTimingNumber(wallRunMsStats, 'median'),
|
|
1092
1353
|
modelLoadMs: safeModelLoadMs,
|
|
1093
1354
|
throughput: {
|
|
1094
1355
|
tokensPerSec: tokensPerSecStats,
|
|
@@ -1098,6 +1359,7 @@ async function runBenchSuite(options = {}) {
|
|
|
1098
1359
|
},
|
|
1099
1360
|
latency: {
|
|
1100
1361
|
totalMs: totalMsStats,
|
|
1362
|
+
wallRunMs: wallRunMsStats,
|
|
1101
1363
|
prefillMs: prefillMsStats,
|
|
1102
1364
|
decodeMs: decodeMsStats,
|
|
1103
1365
|
firstTokenMs: ttftMsStats,
|
|
@@ -1109,10 +1371,13 @@ async function runBenchSuite(options = {}) {
|
|
|
1109
1371
|
},
|
|
1110
1372
|
gpu: gpuPhaseStats,
|
|
1111
1373
|
batching: batchingPhaseStats,
|
|
1374
|
+
decodeCadence: resolveDecodeCadence(getRuntimeConfig(), lastExecutionPlan),
|
|
1112
1375
|
plePreparedTokenCache: plePreparedTokenCacheStats,
|
|
1113
1376
|
decodeMode: lastDecodeMode,
|
|
1114
1377
|
batchGuardReason: lastBatchGuardReason,
|
|
1378
|
+
executionPlan: lastExecutionPlan,
|
|
1115
1379
|
generatedText,
|
|
1380
|
+
referenceTranscript: generatedReferenceTranscript,
|
|
1116
1381
|
promptInput: generatedPromptInput,
|
|
1117
1382
|
};
|
|
1118
1383
|
|
|
@@ -1139,6 +1404,16 @@ async function runBenchSuite(options = {}) {
|
|
|
1139
1404
|
const memoryStats = typeof harness.pipeline?.getMemoryStats === 'function'
|
|
1140
1405
|
? harness.pipeline.getMemoryStats()
|
|
1141
1406
|
: null;
|
|
1407
|
+
const loadTimings = resolvePipelineLoadTimings(harness.pipeline);
|
|
1408
|
+
const loadDiagnostics = buildLoadTimingDiagnostics(
|
|
1409
|
+
safeModelLoadMs,
|
|
1410
|
+
loadTimings.loadTiming,
|
|
1411
|
+
loadTimings.pipelineLoadTiming
|
|
1412
|
+
);
|
|
1413
|
+
const decodeBottleneck = buildDecodeBottleneckDiagnostics(metrics, timing);
|
|
1414
|
+
if (decodeBottleneck) {
|
|
1415
|
+
metrics.decodeBottleneck = decodeBottleneck;
|
|
1416
|
+
}
|
|
1142
1417
|
|
|
1143
1418
|
if (typeof harness.pipeline.unload === 'function' && !options.keepPipeline) {
|
|
1144
1419
|
await harness.pipeline.unload();
|
|
@@ -1148,13 +1423,22 @@ async function runBenchSuite(options = {}) {
|
|
|
1148
1423
|
const timingDiagnostics = buildTimingDiagnostics(timing, {
|
|
1149
1424
|
source: 'doppler',
|
|
1150
1425
|
prefillSemantics: 'internal_prefill_phase',
|
|
1426
|
+
loadTiming: loadTimings.loadTiming,
|
|
1427
|
+
pipelineLoadTiming: loadTimings.pipelineLoadTiming,
|
|
1151
1428
|
});
|
|
1429
|
+
if (decodeBottleneck) {
|
|
1430
|
+
timingDiagnostics.decodeBottleneck = decodeBottleneck;
|
|
1431
|
+
}
|
|
1152
1432
|
const firstLoad = buildFirstLoadComposition({
|
|
1153
1433
|
modelLoadMs: timing.modelLoadMs,
|
|
1154
1434
|
firstTokenMs: timing.firstTokenMs,
|
|
1155
1435
|
firstResponseMs: timing.firstResponseMs,
|
|
1156
1436
|
});
|
|
1157
|
-
const metricsWithContracts = buildSuiteContractMetrics(
|
|
1437
|
+
const metricsWithContracts = buildSuiteContractMetrics(
|
|
1438
|
+
'bench',
|
|
1439
|
+
loadDiagnostics ? { ...metrics, load: loadDiagnostics } : metrics,
|
|
1440
|
+
harness.manifest
|
|
1441
|
+
);
|
|
1158
1442
|
return {
|
|
1159
1443
|
...summary,
|
|
1160
1444
|
modelId: options.modelId || harness.manifest?.modelId || 'unknown',
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* Decode ring buffer pool for GPU batch decode.
|
|
3
3
|
*
|
|
4
4
|
* Preallocates fixed GPUBuffer rings for token outputs, stop flags, and staging readback.
|
|
5
|
+
* Token staging buffers reserve trailing bytes for packed finiteness status.
|
|
5
6
|
* Ring sizing derives from runtime batching config (batchSize × readbackInterval).
|
|
6
7
|
*/
|
|
7
8
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { getDevice, getDeviceLimits } from '../gpu/device.js';
|
|
2
2
|
|
|
3
3
|
const TOKEN_BYTES = 4;
|
|
4
|
+
const FINITENESS_STATUS_BYTES = 16;
|
|
4
5
|
|
|
5
6
|
function clampRingSize(size) {
|
|
6
7
|
if (size == null) return 0;
|
|
@@ -124,13 +125,14 @@ export class DecodeRing {
|
|
|
124
125
|
|
|
125
126
|
const tokensBytes = (normalized.tokensPerInterval + 1) * TOKEN_BYTES;
|
|
126
127
|
const stopBytes = (normalized.tokensPerInterval + 1) * TOKEN_BYTES;
|
|
127
|
-
const
|
|
128
|
+
const tokenStagingBytes = normalized.tokensPerInterval * TOKEN_BYTES + FINITENESS_STATUS_BYTES;
|
|
129
|
+
const stopStagingBytes = normalized.tokensPerInterval * TOKEN_BYTES;
|
|
128
130
|
|
|
129
131
|
assertBufferFits('tokens', tokensBytes, true, limits);
|
|
130
|
-
assertBufferFits('stagingTokens',
|
|
132
|
+
assertBufferFits('stagingTokens', tokenStagingBytes, false, limits);
|
|
131
133
|
if (normalized.stopCheckMode === 'per-token') {
|
|
132
134
|
assertBufferFits('stop', stopBytes, true, limits);
|
|
133
|
-
assertBufferFits('stagingStop',
|
|
135
|
+
assertBufferFits('stagingStop', stopStagingBytes, false, limits);
|
|
134
136
|
}
|
|
135
137
|
|
|
136
138
|
const buffers = {
|
|
@@ -155,7 +157,7 @@ export class DecodeRing {
|
|
|
155
157
|
buffers.stagingTokens = Array.from({ length: normalized.ringStaging }, (_, i) => (
|
|
156
158
|
device.createBuffer({
|
|
157
159
|
label: `decode_ring_staging_tokens_${i}`,
|
|
158
|
-
size:
|
|
160
|
+
size: tokenStagingBytes,
|
|
159
161
|
usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
|
|
160
162
|
})
|
|
161
163
|
));
|
|
@@ -175,7 +177,7 @@ export class DecodeRing {
|
|
|
175
177
|
buffers.stagingStop = Array.from({ length: normalized.ringStaging }, (_, i) => (
|
|
176
178
|
device.createBuffer({
|
|
177
179
|
label: `decode_ring_staging_stop_${i}`,
|
|
178
|
-
size:
|
|
180
|
+
size: stopStagingBytes,
|
|
179
181
|
usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
|
|
180
182
|
})
|
|
181
183
|
));
|
|
@@ -94,6 +94,31 @@ export class KVCache {
|
|
|
94
94
|
tokenIds?: number[] | null
|
|
95
95
|
): void | Promise<void>;
|
|
96
96
|
|
|
97
|
+
/**
|
|
98
|
+
* Record direct f32 source to f16 GPU KV cache update.
|
|
99
|
+
* Does NOT submit - caller is responsible for submitting the recorder.
|
|
100
|
+
*/
|
|
101
|
+
recordUpdateF32ToF16FromGPU(
|
|
102
|
+
recorder: import('../../gpu/kernel-selector.js').CommandRecorder,
|
|
103
|
+
layerIdx: number,
|
|
104
|
+
keysBuffer: GPUBuffer,
|
|
105
|
+
valuesBuffer: GPUBuffer,
|
|
106
|
+
startPos: number,
|
|
107
|
+
numTokens: number,
|
|
108
|
+
tokenIds?: number[] | null
|
|
109
|
+
): void | Promise<void>;
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Record metadata for a fused projection kernel that already wrote f16 K/V
|
|
113
|
+
* into the contiguous GPU cache through storage-buffer side effects.
|
|
114
|
+
*/
|
|
115
|
+
recordF16UpdateAlreadyWrittenFromGPU(
|
|
116
|
+
layerIdx: number,
|
|
117
|
+
startPos: number,
|
|
118
|
+
numTokens: number,
|
|
119
|
+
tokenIds?: number[] | null
|
|
120
|
+
): void;
|
|
121
|
+
|
|
97
122
|
/**
|
|
98
123
|
* Get cached keys and values for a layer
|
|
99
124
|
*/
|