doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -24,7 +24,16 @@ export declare function sampleTimingNumber(
|
|
|
24
24
|
fallback?: number | null
|
|
25
25
|
): number | null;
|
|
26
26
|
export declare function buildCanonicalTiming(overrides?: Record<string, unknown>): Record<string, unknown>;
|
|
27
|
+
export declare function buildLoadTimingDiagnostics(
|
|
28
|
+
modelLoadMs: unknown,
|
|
29
|
+
loadTiming?: Record<string, unknown> | null,
|
|
30
|
+
pipelineLoadTiming?: Record<string, unknown> | null
|
|
31
|
+
): Record<string, unknown> | null;
|
|
27
32
|
export declare function buildTimingDiagnostics(timing?: Record<string, unknown>, options?: Record<string, unknown>): Record<string, unknown>;
|
|
33
|
+
export declare function buildDecodeBottleneckDiagnostics(
|
|
34
|
+
metrics?: Record<string, unknown>,
|
|
35
|
+
timing?: Record<string, unknown>
|
|
36
|
+
): Record<string, unknown> | null;
|
|
28
37
|
|
|
29
38
|
export declare function buildFirstLoadComposition(
|
|
30
39
|
fields?: Record<string, unknown>
|
|
@@ -56,6 +56,7 @@ export function buildDiffusionPerformanceArtifact({
|
|
|
56
56
|
avgDecodeTokens,
|
|
57
57
|
cpuStats,
|
|
58
58
|
gpuStats,
|
|
59
|
+
modality = 'image',
|
|
59
60
|
}) {
|
|
60
61
|
const cpuPrefillMs = safeStatsValue(cpuStats?.prefillMs?.median);
|
|
61
62
|
const cpuDenoiseMs = safeStatsValue(cpuStats?.denoiseMs?.median);
|
|
@@ -73,6 +74,7 @@ export function buildDiffusionPerformanceArtifact({
|
|
|
73
74
|
schemaVersion: 1,
|
|
74
75
|
warmupRuns,
|
|
75
76
|
timedRuns,
|
|
77
|
+
modality,
|
|
76
78
|
shape: {
|
|
77
79
|
width,
|
|
78
80
|
height,
|
|
@@ -203,8 +205,197 @@ export function buildCanonicalTiming(overrides = {}) {
|
|
|
203
205
|
};
|
|
204
206
|
}
|
|
205
207
|
|
|
208
|
+
function normalizeLoadTimingInteger(value) {
|
|
209
|
+
return Number.isFinite(value) ? Math.max(0, Math.round(value)) : null;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function normalizeLoadTimingMs(value) {
|
|
213
|
+
return Number.isFinite(value) ? toTimingNumber(value, null) : null;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function normalizeLoadTimingString(value) {
|
|
217
|
+
return typeof value === 'string' && value.length > 0 ? value : null;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function normalizeLoaderLoadTiming(loadTiming) {
|
|
221
|
+
if (!loadTiming || typeof loadTiming !== 'object' || Array.isArray(loadTiming)) {
|
|
222
|
+
return null;
|
|
223
|
+
}
|
|
224
|
+
const rawPhasesMs = loadTiming.phasesMs && typeof loadTiming.phasesMs === 'object'
|
|
225
|
+
? loadTiming.phasesMs
|
|
226
|
+
: {};
|
|
227
|
+
const phasesMs = {
|
|
228
|
+
preflight: normalizeLoadTimingMs(rawPhasesMs.preflight),
|
|
229
|
+
tensorLocations: normalizeLoadTimingMs(rawPhasesMs.tensorLocations),
|
|
230
|
+
embeddings: normalizeLoadTimingMs(rawPhasesMs.embeddings),
|
|
231
|
+
layers: normalizeLoadTimingMs(rawPhasesMs.layers),
|
|
232
|
+
finalWeights: normalizeLoadTimingMs(rawPhasesMs.finalWeights),
|
|
233
|
+
cleanup: normalizeLoadTimingMs(rawPhasesMs.cleanup),
|
|
234
|
+
};
|
|
235
|
+
const rawLayers = loadTiming.layers && typeof loadTiming.layers === 'object'
|
|
236
|
+
? loadTiming.layers
|
|
237
|
+
: {};
|
|
238
|
+
const status = typeof loadTiming.status === 'string' && loadTiming.status.length > 0
|
|
239
|
+
? loadTiming.status
|
|
240
|
+
: null;
|
|
241
|
+
return {
|
|
242
|
+
schemaVersion: Number.isInteger(loadTiming.schemaVersion) ? loadTiming.schemaVersion : 1,
|
|
243
|
+
source: typeof loadTiming.source === 'string' && loadTiming.source.length > 0
|
|
244
|
+
? loadTiming.source
|
|
245
|
+
: 'doppler-loader',
|
|
246
|
+
modelId: typeof loadTiming.modelId === 'string' ? loadTiming.modelId : null,
|
|
247
|
+
status,
|
|
248
|
+
customShardLoader: loadTiming.customShardLoader === true,
|
|
249
|
+
byteAccountingMode: typeof loadTiming.byteAccountingMode === 'string' && loadTiming.byteAccountingMode.length > 0
|
|
250
|
+
? loadTiming.byteAccountingMode
|
|
251
|
+
: (loadTiming.customShardLoader === true ? 'custom-loader-progress-unavailable' : 'full-shard-progress'),
|
|
252
|
+
totalBytes: normalizeLoadTimingInteger(loadTiming.totalBytes),
|
|
253
|
+
totalShards: normalizeLoadTimingInteger(loadTiming.totalShards),
|
|
254
|
+
bytesLoaded: normalizeLoadTimingInteger(loadTiming.bytesLoaded),
|
|
255
|
+
shardsLoaded: normalizeLoadTimingInteger(loadTiming.shardsLoaded),
|
|
256
|
+
bytesPerSecond: normalizeLoadTimingInteger(loadTiming.bytesPerSecond),
|
|
257
|
+
phasesMs,
|
|
258
|
+
layers: {
|
|
259
|
+
count: normalizeLoadTimingInteger(rawLayers.count),
|
|
260
|
+
totalMs: normalizeLoadTimingMs(rawLayers.totalMs),
|
|
261
|
+
meanMs: normalizeLoadTimingMs(rawLayers.meanMs),
|
|
262
|
+
maxMs: normalizeLoadTimingMs(rawLayers.maxMs),
|
|
263
|
+
maxLayer: normalizeLoadTimingInteger(rawLayers.maxLayer),
|
|
264
|
+
},
|
|
265
|
+
totalMs: normalizeLoadTimingMs(loadTiming.totalMs),
|
|
266
|
+
failedPhase: typeof loadTiming.failedPhase === 'string' ? loadTiming.failedPhase : null,
|
|
267
|
+
error: typeof loadTiming.error === 'string' ? loadTiming.error : null,
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
function normalizeTokenizerLoadTiming(tokenizerLoadTiming) {
|
|
272
|
+
if (!tokenizerLoadTiming || typeof tokenizerLoadTiming !== 'object' || Array.isArray(tokenizerLoadTiming)) {
|
|
273
|
+
return null;
|
|
274
|
+
}
|
|
275
|
+
const rawPhasesMs = tokenizerLoadTiming.phasesMs && typeof tokenizerLoadTiming.phasesMs === 'object'
|
|
276
|
+
? tokenizerLoadTiming.phasesMs
|
|
277
|
+
: {};
|
|
278
|
+
return {
|
|
279
|
+
schemaVersion: Number.isInteger(tokenizerLoadTiming.schemaVersion) ? tokenizerLoadTiming.schemaVersion : 1,
|
|
280
|
+
source: normalizeLoadTimingString(tokenizerLoadTiming.source) ?? 'doppler-tokenizer',
|
|
281
|
+
modelId: typeof tokenizerLoadTiming.modelId === 'string' ? tokenizerLoadTiming.modelId : null,
|
|
282
|
+
status: normalizeLoadTimingString(tokenizerLoadTiming.status),
|
|
283
|
+
tokenizerType: normalizeLoadTimingString(tokenizerLoadTiming.tokenizerType),
|
|
284
|
+
tokenizerFile: normalizeLoadTimingString(tokenizerLoadTiming.tokenizerFile),
|
|
285
|
+
backend: normalizeLoadTimingString(tokenizerLoadTiming.backend),
|
|
286
|
+
assetSource: normalizeLoadTimingString(tokenizerLoadTiming.assetSource),
|
|
287
|
+
cacheHit: tokenizerLoadTiming.cacheHit === true,
|
|
288
|
+
phasesMs: {
|
|
289
|
+
configResolution: normalizeLoadTimingMs(rawPhasesMs.configResolution),
|
|
290
|
+
cacheLookup: normalizeLoadTimingMs(rawPhasesMs.cacheLookup),
|
|
291
|
+
backendCreate: normalizeLoadTimingMs(rawPhasesMs.backendCreate),
|
|
292
|
+
assetLoad: normalizeLoadTimingMs(rawPhasesMs.assetLoad),
|
|
293
|
+
assetParse: normalizeLoadTimingMs(rawPhasesMs.assetParse),
|
|
294
|
+
backendLoad: normalizeLoadTimingMs(rawPhasesMs.backendLoad),
|
|
295
|
+
cacheStore: normalizeLoadTimingMs(rawPhasesMs.cacheStore),
|
|
296
|
+
},
|
|
297
|
+
totalMs: normalizeLoadTimingMs(tokenizerLoadTiming.totalMs),
|
|
298
|
+
error: typeof tokenizerLoadTiming.error === 'string' ? tokenizerLoadTiming.error : null,
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
function normalizePipelineLoadTiming(pipelineLoadTiming) {
|
|
303
|
+
if (!pipelineLoadTiming || typeof pipelineLoadTiming !== 'object' || Array.isArray(pipelineLoadTiming)) {
|
|
304
|
+
return null;
|
|
305
|
+
}
|
|
306
|
+
const rawPhasesMs = pipelineLoadTiming.phasesMs && typeof pipelineLoadTiming.phasesMs === 'object'
|
|
307
|
+
? pipelineLoadTiming.phasesMs
|
|
308
|
+
: {};
|
|
309
|
+
const phasesMs = {
|
|
310
|
+
reset: normalizeLoadTimingMs(rawPhasesMs.reset),
|
|
311
|
+
configResolution: normalizeLoadTimingMs(rawPhasesMs.configResolution),
|
|
312
|
+
kernelWarmup: normalizeLoadTimingMs(rawPhasesMs.kernelWarmup),
|
|
313
|
+
tokenizer: normalizeLoadTimingMs(rawPhasesMs.tokenizer),
|
|
314
|
+
executionSetup: normalizeLoadTimingMs(rawPhasesMs.executionSetup),
|
|
315
|
+
loadWeights: normalizeLoadTimingMs(rawPhasesMs.loadWeights),
|
|
316
|
+
rope: normalizeLoadTimingMs(rawPhasesMs.rope),
|
|
317
|
+
convStates: normalizeLoadTimingMs(rawPhasesMs.convStates),
|
|
318
|
+
};
|
|
319
|
+
return {
|
|
320
|
+
schemaVersion: Number.isInteger(pipelineLoadTiming.schemaVersion) ? pipelineLoadTiming.schemaVersion : 1,
|
|
321
|
+
source: typeof pipelineLoadTiming.source === 'string' && pipelineLoadTiming.source.length > 0
|
|
322
|
+
? pipelineLoadTiming.source
|
|
323
|
+
: 'doppler-pipeline',
|
|
324
|
+
modelId: typeof pipelineLoadTiming.modelId === 'string' ? pipelineLoadTiming.modelId : null,
|
|
325
|
+
status: typeof pipelineLoadTiming.status === 'string' && pipelineLoadTiming.status.length > 0
|
|
326
|
+
? pipelineLoadTiming.status
|
|
327
|
+
: null,
|
|
328
|
+
phasesMs,
|
|
329
|
+
details: {
|
|
330
|
+
tokenizer: normalizeTokenizerLoadTiming(pipelineLoadTiming.details?.tokenizer),
|
|
331
|
+
},
|
|
332
|
+
totalMs: normalizeLoadTimingMs(pipelineLoadTiming.totalMs),
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
export function buildLoadTimingDiagnostics(modelLoadMs, loadTiming, pipelineLoadTiming = null) {
|
|
337
|
+
const loader = normalizeLoaderLoadTiming(loadTiming);
|
|
338
|
+
const pipeline = normalizePipelineLoadTiming(pipelineLoadTiming);
|
|
339
|
+
if (!loader && !pipeline) {
|
|
340
|
+
return null;
|
|
341
|
+
}
|
|
342
|
+
const normalizedModelLoadMs = Number.isFinite(modelLoadMs)
|
|
343
|
+
? toTimingNumber(modelLoadMs, null)
|
|
344
|
+
: null;
|
|
345
|
+
const modelLoadMinusLoaderMs = (
|
|
346
|
+
Number.isFinite(normalizedModelLoadMs)
|
|
347
|
+
&& Number.isFinite(loader?.totalMs)
|
|
348
|
+
)
|
|
349
|
+
? toTimingNumber(normalizedModelLoadMs - loader.totalMs, null)
|
|
350
|
+
: null;
|
|
351
|
+
const modelLoadMinusPipelineMs = (
|
|
352
|
+
Number.isFinite(normalizedModelLoadMs)
|
|
353
|
+
&& Number.isFinite(pipeline?.totalMs)
|
|
354
|
+
)
|
|
355
|
+
? toTimingNumber(normalizedModelLoadMs - pipeline.totalMs, null)
|
|
356
|
+
: null;
|
|
357
|
+
const pipelineMinusLoaderMs = (
|
|
358
|
+
Number.isFinite(pipeline?.totalMs)
|
|
359
|
+
&& Number.isFinite(loader?.totalMs)
|
|
360
|
+
)
|
|
361
|
+
? toTimingNumber(pipeline.totalMs - loader.totalMs, null)
|
|
362
|
+
: null;
|
|
363
|
+
return {
|
|
364
|
+
schemaVersion: 1,
|
|
365
|
+
source: 'doppler',
|
|
366
|
+
semantics: {
|
|
367
|
+
modelLoadMs: 'suite initialization time before generation',
|
|
368
|
+
loaderTotalMs: 'DopplerLoader.load() weight-loader wall time',
|
|
369
|
+
pipelineTotalMs: 'InferencePipeline.loadModel() wall time',
|
|
370
|
+
modelLoadMinusPipelineMs: 'modelLoadMs - pipeline.totalMs; harness, storage, manifest, GPU init, and pipeline construction',
|
|
371
|
+
pipelineMinusLoaderMs: 'pipeline.totalMs - loader.totalMs; config, tokenizer, KV, RoPE, and other pipeline setup',
|
|
372
|
+
modelLoadMinusLoaderMs: 'modelLoadMs - loader.totalMs',
|
|
373
|
+
},
|
|
374
|
+
modelLoadMs: normalizedModelLoadMs,
|
|
375
|
+
loader,
|
|
376
|
+
pipeline,
|
|
377
|
+
residualsMs: {
|
|
378
|
+
modelLoadMinusLoaderMs,
|
|
379
|
+
modelLoadMinusPipelineMs,
|
|
380
|
+
pipelineMinusLoaderMs,
|
|
381
|
+
},
|
|
382
|
+
consistent: {
|
|
383
|
+
loaderWithinModelLoad: Number.isFinite(modelLoadMinusLoaderMs)
|
|
384
|
+
? modelLoadMinusLoaderMs >= -2
|
|
385
|
+
: null,
|
|
386
|
+
pipelineWithinModelLoad: Number.isFinite(modelLoadMinusPipelineMs)
|
|
387
|
+
? modelLoadMinusPipelineMs >= -2
|
|
388
|
+
: null,
|
|
389
|
+
loaderWithinPipeline: Number.isFinite(pipelineMinusLoaderMs)
|
|
390
|
+
? pipelineMinusLoaderMs >= -2
|
|
391
|
+
: null,
|
|
392
|
+
},
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
|
|
206
396
|
export function buildTimingDiagnostics(timing = {}, options = {}) {
|
|
207
397
|
const prefillSemantics = String(options.prefillSemantics || 'internal_prefill_phase');
|
|
398
|
+
const decodeSemantics = String(options.decodeSemantics || 'time after first token');
|
|
208
399
|
const source = String(options.source || 'doppler');
|
|
209
400
|
const modelLoadMs = Number.isFinite(timing.modelLoadMs) ? toTimingNumber(timing.modelLoadMs) : null;
|
|
210
401
|
const firstTokenMs = Number.isFinite(timing.firstTokenMs) ? toTimingNumber(timing.firstTokenMs) : null;
|
|
@@ -235,7 +426,7 @@ export function buildTimingDiagnostics(timing = {}, options = {}) {
|
|
|
235
426
|
? toTimingNumber(totalRunMs - runFromPrefillAndDecodeMs)
|
|
236
427
|
: null;
|
|
237
428
|
|
|
238
|
-
|
|
429
|
+
const diagnostics = {
|
|
239
430
|
schemaVersion: 1,
|
|
240
431
|
source,
|
|
241
432
|
semantics: {
|
|
@@ -243,7 +434,7 @@ export function buildTimingDiagnostics(timing = {}, options = {}) {
|
|
|
243
434
|
firstTokenMs: 'ttft from generation start',
|
|
244
435
|
firstResponseMs: 'modelLoadMs + firstTokenMs',
|
|
245
436
|
prefillMs: prefillSemantics,
|
|
246
|
-
decodeMs:
|
|
437
|
+
decodeMs: decodeSemantics,
|
|
247
438
|
totalRunMs: 'prefillMs + decodeMs',
|
|
248
439
|
},
|
|
249
440
|
componentsMs: {
|
|
@@ -267,6 +458,251 @@ export function buildTimingDiagnostics(timing = {}, options = {}) {
|
|
|
267
458
|
totalRun: Number.isFinite(runResidualMs) ? Math.abs(runResidualMs) <= 2 : null,
|
|
268
459
|
},
|
|
269
460
|
};
|
|
461
|
+
const load = buildLoadTimingDiagnostics(modelLoadMs, options.loadTiming, options.pipelineLoadTiming);
|
|
462
|
+
if (load) {
|
|
463
|
+
diagnostics.load = load;
|
|
464
|
+
}
|
|
465
|
+
return diagnostics;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
function metricNumber(value) {
|
|
469
|
+
if (Number.isFinite(value)) {
|
|
470
|
+
return Number(value);
|
|
471
|
+
}
|
|
472
|
+
if (
|
|
473
|
+
value
|
|
474
|
+
&& typeof value === 'object'
|
|
475
|
+
&& (
|
|
476
|
+
(Number.isFinite(value.samplesAfterOutlierRemoval) && value.samplesAfterOutlierRemoval <= 0)
|
|
477
|
+
|| (Number.isFinite(value.samples) && value.samples <= 0)
|
|
478
|
+
)
|
|
479
|
+
) {
|
|
480
|
+
return null;
|
|
481
|
+
}
|
|
482
|
+
if (value && typeof value === 'object' && Number.isFinite(value.median)) {
|
|
483
|
+
return Number(value.median);
|
|
484
|
+
}
|
|
485
|
+
return null;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
function shareOfDecode(value, decodeWallMs) {
|
|
489
|
+
if (!Number.isFinite(value) || !Number.isFinite(decodeWallMs) || decodeWallMs <= 0) {
|
|
490
|
+
return null;
|
|
491
|
+
}
|
|
492
|
+
return Number((value / decodeWallMs).toFixed(4));
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
function nullableTimingNumber(value) {
|
|
496
|
+
return Number.isFinite(value) ? toTimingNumber(value, null) : null;
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
function bottleneckClass(componentId) {
|
|
500
|
+
if (componentId === 'command_record') {
|
|
501
|
+
return 'command-record';
|
|
502
|
+
}
|
|
503
|
+
if (
|
|
504
|
+
componentId === 'submit_readback_wait'
|
|
505
|
+
|| componentId === 'submit_readback_slack'
|
|
506
|
+
|| componentId === 'readback_map_wait'
|
|
507
|
+
|| componentId === 'readback_cleanup'
|
|
508
|
+
|| componentId === 'readback_copy'
|
|
509
|
+
|| componentId === 'submit_readback_unattributed'
|
|
510
|
+
) {
|
|
511
|
+
return 'submit-readback-wait';
|
|
512
|
+
}
|
|
513
|
+
if (componentId === 'gpu_compute') {
|
|
514
|
+
return 'gpu-compute';
|
|
515
|
+
}
|
|
516
|
+
if (componentId === 'orchestration') {
|
|
517
|
+
return 'orchestration';
|
|
518
|
+
}
|
|
519
|
+
if (componentId === 'unattributed') {
|
|
520
|
+
return 'unattributed';
|
|
521
|
+
}
|
|
522
|
+
return null;
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
function normalizeTopOps(entries) {
|
|
526
|
+
if (!Array.isArray(entries)) {
|
|
527
|
+
return [];
|
|
528
|
+
}
|
|
529
|
+
return entries
|
|
530
|
+
.slice(0, 8)
|
|
531
|
+
.map((entry) => ({
|
|
532
|
+
label: typeof entry?.label === 'string' && entry.label.length > 0 ? entry.label : null,
|
|
533
|
+
count: Number.isFinite(entry?.count) ? Number(entry.count) : null,
|
|
534
|
+
shareOfOps: Number.isFinite(entry?.shareOfOps) ? Number(entry.shareOfOps) : null,
|
|
535
|
+
}))
|
|
536
|
+
.filter((entry) => entry.label || entry.count != null || entry.shareOfOps != null);
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
function normalizeUniformCacheForDiagnostics(value) {
|
|
540
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
541
|
+
return null;
|
|
542
|
+
}
|
|
543
|
+
const normalized = {};
|
|
544
|
+
for (const key of [
|
|
545
|
+
'hits',
|
|
546
|
+
'misses',
|
|
547
|
+
'totalLookups',
|
|
548
|
+
'hitRateRatio',
|
|
549
|
+
'evictions',
|
|
550
|
+
'currentSize',
|
|
551
|
+
'pendingDestruction',
|
|
552
|
+
]) {
|
|
553
|
+
if (Number.isFinite(value[key])) {
|
|
554
|
+
normalized[key] = Number(value[key]);
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
if (typeof value.hitRate === 'string' && value.hitRate.length > 0) {
|
|
558
|
+
normalized.hitRate = value.hitRate;
|
|
559
|
+
}
|
|
560
|
+
return Object.keys(normalized).length > 0 ? normalized : null;
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
export function buildDecodeBottleneckDiagnostics(metrics = {}, timing = {}) {
|
|
564
|
+
const gpu = metrics?.gpu && typeof metrics.gpu === 'object' ? metrics.gpu : {};
|
|
565
|
+
const decodeWallMs = metricNumber(metrics?.decodeMs)
|
|
566
|
+
?? metricNumber(metrics?.latency?.decodeMs)
|
|
567
|
+
?? metricNumber(timing?.decodeMs);
|
|
568
|
+
if (!Number.isFinite(decodeWallMs) || decodeWallMs <= 0) {
|
|
569
|
+
return null;
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
const commandRecordMs = metricNumber(gpu.decodeRecordMs);
|
|
573
|
+
const commandRecordOps = metricNumber(gpu.decodeRecordOps);
|
|
574
|
+
const commandRecordPasses = metricNumber(gpu.decodeRecordPasses);
|
|
575
|
+
const commandRecordMsPerOp = metricNumber(gpu.decodeRecordMsPerOp);
|
|
576
|
+
const commandRecordMsPerPass = metricNumber(gpu.decodeRecordMsPerPass);
|
|
577
|
+
const commandRecordPassesPerOp = metricNumber(gpu.decodeRecordPassesPerOp);
|
|
578
|
+
const commandRecordMsPerExecutedBatchToken = metricNumber(gpu.decodeRecordMsPerExecutedBatchToken);
|
|
579
|
+
const commandRecordOpsPerExecutedBatchToken = metricNumber(gpu.decodeRecordOpsPerExecutedBatchToken);
|
|
580
|
+
const commandRecordPassesPerExecutedBatchToken = metricNumber(gpu.decodeRecordPassesPerExecutedBatchToken);
|
|
581
|
+
const commandRecordUniqueOpLabels = metricNumber(gpu.decodeRecordUniqueOpLabels);
|
|
582
|
+
const submitWaitMs = metricNumber(gpu.decodeSubmitWaitMs);
|
|
583
|
+
const readbackWaitMs = metricNumber(gpu.decodeReadbackWaitMs);
|
|
584
|
+
const readbackMapWaitMs = metricNumber(gpu.decodeReadbackMapWaitMs);
|
|
585
|
+
const readbackCleanupMs = metricNumber(gpu.decodeReadbackCleanupMs);
|
|
586
|
+
const readbackCopyMs = metricNumber(gpu.decodeReadbackCopyMs);
|
|
587
|
+
const orchestrationMs = metricNumber(gpu.decodeOrchestrationMs);
|
|
588
|
+
const gpuTimestampMs = metricNumber(gpu.decodeMs);
|
|
589
|
+
const uniformCache = normalizeUniformCacheForDiagnostics(gpu.uniformCache);
|
|
590
|
+
const waitCandidates = [submitWaitMs, readbackWaitMs].filter(Number.isFinite);
|
|
591
|
+
const effectiveSubmitReadbackWaitMs = waitCandidates.length > 0 ? Math.max(...waitCandidates) : null;
|
|
592
|
+
const submitReadbackSlackMs = (
|
|
593
|
+
Number.isFinite(effectiveSubmitReadbackWaitMs)
|
|
594
|
+
&& Number.isFinite(gpuTimestampMs)
|
|
595
|
+
)
|
|
596
|
+
? Math.max(0, effectiveSubmitReadbackWaitMs - gpuTimestampMs)
|
|
597
|
+
: null;
|
|
598
|
+
const readbackSubcomponentMs = [
|
|
599
|
+
readbackMapWaitMs,
|
|
600
|
+
readbackCleanupMs,
|
|
601
|
+
readbackCopyMs,
|
|
602
|
+
]
|
|
603
|
+
.filter(Number.isFinite)
|
|
604
|
+
.reduce((sum, value) => sum + value, 0);
|
|
605
|
+
const hasReadbackSubcomponents = readbackSubcomponentMs > 0;
|
|
606
|
+
const readbackUnattributedMs = (
|
|
607
|
+
hasReadbackSubcomponents
|
|
608
|
+
&& Number.isFinite(effectiveSubmitReadbackWaitMs)
|
|
609
|
+
)
|
|
610
|
+
? Math.max(0, effectiveSubmitReadbackWaitMs - readbackSubcomponentMs)
|
|
611
|
+
: null;
|
|
612
|
+
const accountedMs = [
|
|
613
|
+
commandRecordMs,
|
|
614
|
+
effectiveSubmitReadbackWaitMs,
|
|
615
|
+
orchestrationMs,
|
|
616
|
+
]
|
|
617
|
+
.filter(Number.isFinite)
|
|
618
|
+
.reduce((sum, value) => sum + value, 0);
|
|
619
|
+
const residualMs = Math.max(0, decodeWallMs - accountedMs);
|
|
620
|
+
const components = [
|
|
621
|
+
{ id: 'command_record', label: 'command recording', ms: commandRecordMs },
|
|
622
|
+
{
|
|
623
|
+
id: Number.isFinite(gpuTimestampMs) ? 'submit_readback_slack' : 'submit_readback_wait',
|
|
624
|
+
label: Number.isFinite(gpuTimestampMs) ? 'submit/readback slack' : 'submit/readback wait',
|
|
625
|
+
ms: hasReadbackSubcomponents
|
|
626
|
+
? null
|
|
627
|
+
: (Number.isFinite(gpuTimestampMs) ? submitReadbackSlackMs : effectiveSubmitReadbackWaitMs),
|
|
628
|
+
},
|
|
629
|
+
{ id: 'readback_map_wait', label: 'readback map wait', ms: readbackMapWaitMs },
|
|
630
|
+
{ id: 'readback_cleanup', label: 'readback cleanup', ms: readbackCleanupMs },
|
|
631
|
+
{ id: 'readback_copy', label: 'readback CPU copy', ms: readbackCopyMs },
|
|
632
|
+
{ id: 'submit_readback_unattributed', label: 'submit/readback unattributed', ms: readbackUnattributedMs },
|
|
633
|
+
{ id: 'gpu_compute', label: 'GPU timestamp work', ms: gpuTimestampMs },
|
|
634
|
+
{ id: 'orchestration', label: 'decode orchestration', ms: orchestrationMs },
|
|
635
|
+
{ id: 'unattributed', label: 'unattributed decode wall', ms: residualMs },
|
|
636
|
+
].filter((component) => Number.isFinite(component.ms) && component.ms > 0);
|
|
637
|
+
const dominant = components.length > 0
|
|
638
|
+
? components.reduce((best, component) => component.ms > best.ms ? component : best, components[0])
|
|
639
|
+
: null;
|
|
640
|
+
const dominantRecord = dominant
|
|
641
|
+
? {
|
|
642
|
+
id: dominant.id,
|
|
643
|
+
label: dominant.label,
|
|
644
|
+
ms: nullableTimingNumber(dominant.ms),
|
|
645
|
+
shareOfDecode: shareOfDecode(dominant.ms, decodeWallMs),
|
|
646
|
+
}
|
|
647
|
+
: null;
|
|
648
|
+
|
|
649
|
+
return {
|
|
650
|
+
schemaVersion: 1,
|
|
651
|
+
source: 'doppler',
|
|
652
|
+
dominant: dominantRecord,
|
|
653
|
+
bottleneckClass: dominantRecord ? bottleneckClass(dominantRecord.id) : null,
|
|
654
|
+
decodeWallMs: nullableTimingNumber(decodeWallMs),
|
|
655
|
+
componentsMs: {
|
|
656
|
+
commandRecordMs: nullableTimingNumber(commandRecordMs),
|
|
657
|
+
submitWaitMs: nullableTimingNumber(submitWaitMs),
|
|
658
|
+
readbackWaitMs: nullableTimingNumber(readbackWaitMs),
|
|
659
|
+
effectiveSubmitReadbackWaitMs: nullableTimingNumber(effectiveSubmitReadbackWaitMs),
|
|
660
|
+
readbackMapWaitMs: nullableTimingNumber(readbackMapWaitMs),
|
|
661
|
+
readbackCleanupMs: nullableTimingNumber(readbackCleanupMs),
|
|
662
|
+
readbackCopyMs: nullableTimingNumber(readbackCopyMs),
|
|
663
|
+
readbackUnattributedMs: nullableTimingNumber(readbackUnattributedMs),
|
|
664
|
+
gpuTimestampMs: nullableTimingNumber(gpuTimestampMs),
|
|
665
|
+
submitReadbackSlackMs: nullableTimingNumber(submitReadbackSlackMs),
|
|
666
|
+
orchestrationMs: nullableTimingNumber(orchestrationMs),
|
|
667
|
+
residualMs: nullableTimingNumber(residualMs),
|
|
668
|
+
},
|
|
669
|
+
recording: {
|
|
670
|
+
opCount: nullableTimingNumber(commandRecordOps),
|
|
671
|
+
passCount: nullableTimingNumber(commandRecordPasses),
|
|
672
|
+
uniqueOpLabels: nullableTimingNumber(commandRecordUniqueOpLabels),
|
|
673
|
+
msPerOp: nullableTimingNumber(commandRecordMsPerOp),
|
|
674
|
+
msPerPass: nullableTimingNumber(commandRecordMsPerPass),
|
|
675
|
+
passesPerOp: nullableTimingNumber(commandRecordPassesPerOp),
|
|
676
|
+
msPerExecutedBatchToken: nullableTimingNumber(commandRecordMsPerExecutedBatchToken),
|
|
677
|
+
opsPerExecutedBatchToken: nullableTimingNumber(commandRecordOpsPerExecutedBatchToken),
|
|
678
|
+
passesPerExecutedBatchToken: nullableTimingNumber(commandRecordPassesPerExecutedBatchToken),
|
|
679
|
+
topOps: normalizeTopOps(gpu.decodeRecordTopOps),
|
|
680
|
+
topOpGroups: normalizeTopOps(gpu.decodeRecordTopOpGroups),
|
|
681
|
+
uniformCache,
|
|
682
|
+
},
|
|
683
|
+
shares: {
|
|
684
|
+
commandRecord: shareOfDecode(commandRecordMs, decodeWallMs),
|
|
685
|
+
submitWait: shareOfDecode(submitWaitMs, decodeWallMs),
|
|
686
|
+
readbackWait: shareOfDecode(readbackWaitMs, decodeWallMs),
|
|
687
|
+
effectiveSubmitReadbackWait: shareOfDecode(effectiveSubmitReadbackWaitMs, decodeWallMs),
|
|
688
|
+
readbackMapWait: shareOfDecode(readbackMapWaitMs, decodeWallMs),
|
|
689
|
+
readbackCleanup: shareOfDecode(readbackCleanupMs, decodeWallMs),
|
|
690
|
+
readbackCopy: shareOfDecode(readbackCopyMs, decodeWallMs),
|
|
691
|
+
readbackUnattributed: shareOfDecode(readbackUnattributedMs, decodeWallMs),
|
|
692
|
+
gpuTimestamp: shareOfDecode(gpuTimestampMs, decodeWallMs),
|
|
693
|
+
submitReadbackSlack: shareOfDecode(submitReadbackSlackMs, decodeWallMs),
|
|
694
|
+
orchestration: shareOfDecode(orchestrationMs, decodeWallMs),
|
|
695
|
+
residual: shareOfDecode(residualMs, decodeWallMs),
|
|
696
|
+
},
|
|
697
|
+
semantics: {
|
|
698
|
+
effectiveSubmitReadbackWaitMs: 'max(decodeSubmitWaitMs, decodeReadbackWaitMs); submit and readback waits overlap.',
|
|
699
|
+
readbackMapWaitMs: 'Wall time awaiting staging-buffer mapAsync; usually includes GPU completion behind the readback.',
|
|
700
|
+
gpuTimestampMs: 'Timestamp-query GPU work when available; null means it was not captured.',
|
|
701
|
+
topOps: 'Highest-count exact compute-pass labels observed during command recording.',
|
|
702
|
+
topOpGroups: 'Highest-count compute-pass labels after grouping repeated per-layer labels.',
|
|
703
|
+
uniformCache: 'Current uniform-buffer cache counters at the end of the measured suite.',
|
|
704
|
+
},
|
|
705
|
+
};
|
|
270
706
|
}
|
|
271
707
|
|
|
272
708
|
// Mirrors `buildFirstLoadComposition` on the transformers.js runner
|
|
@@ -18,11 +18,88 @@ export declare function resolveBenchmarkRunSettings(
|
|
|
18
18
|
sampling: Record<string, unknown>;
|
|
19
19
|
seed?: number;
|
|
20
20
|
};
|
|
21
|
+
export declare function normalizeDecodeRecordOpLabels(
|
|
22
|
+
value: unknown
|
|
23
|
+
): Record<string, number> | null;
|
|
24
|
+
export declare function buildDecodeRecordTopOps(
|
|
25
|
+
labelCounts: unknown,
|
|
26
|
+
totalOps?: number | null,
|
|
27
|
+
limit?: number
|
|
28
|
+
): Array<{
|
|
29
|
+
label: string;
|
|
30
|
+
count: number;
|
|
31
|
+
shareOfOps: number | null;
|
|
32
|
+
}>;
|
|
33
|
+
export declare function groupDecodeRecordOpLabels(
|
|
34
|
+
labelCounts: unknown
|
|
35
|
+
): Record<string, number> | null;
|
|
36
|
+
export declare function buildDecodeRecordTopOpGroups(
|
|
37
|
+
labelCounts: unknown,
|
|
38
|
+
totalOps?: number | null,
|
|
39
|
+
limit?: number
|
|
40
|
+
): Array<{
|
|
41
|
+
label: string;
|
|
42
|
+
count: number;
|
|
43
|
+
shareOfOps: number | null;
|
|
44
|
+
}>;
|
|
45
|
+
export declare function normalizeUniformCacheStats(value: unknown): {
|
|
46
|
+
hits?: number;
|
|
47
|
+
misses?: number;
|
|
48
|
+
totalLookups?: number;
|
|
49
|
+
hitRateRatio?: number;
|
|
50
|
+
hitRate?: string;
|
|
51
|
+
evictions?: number;
|
|
52
|
+
currentSize?: number;
|
|
53
|
+
pendingDestruction?: number;
|
|
54
|
+
} | null;
|
|
21
55
|
export declare function runEmbeddingSemanticChecks(
|
|
22
56
|
pipeline: Record<string, unknown>,
|
|
23
57
|
options?: Record<string, unknown> | null
|
|
24
58
|
): Promise<Record<string, unknown>>;
|
|
25
59
|
export declare function isCoherentOutput(tokens: Array<unknown>, output: unknown): boolean;
|
|
60
|
+
|
|
61
|
+
export interface ReferenceLogitsDigest {
|
|
62
|
+
index: number | null;
|
|
63
|
+
tokenId: number | null;
|
|
64
|
+
inputTokenCount: number | null;
|
|
65
|
+
dtype: 'f32';
|
|
66
|
+
elementCount: number;
|
|
67
|
+
digest: string;
|
|
68
|
+
top?: Array<{
|
|
69
|
+
tokenId: number;
|
|
70
|
+
logit: number;
|
|
71
|
+
text: string | null;
|
|
72
|
+
}>;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export interface KvCacheLayerByteProof {
|
|
76
|
+
layer: number;
|
|
77
|
+
seqLen: number;
|
|
78
|
+
keyBytes: number;
|
|
79
|
+
valueBytes: number;
|
|
80
|
+
keyDigest: string;
|
|
81
|
+
valueDigest: string;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export interface KvCacheByteProof {
|
|
85
|
+
mode: 'sha256-layer-kv-bytes';
|
|
86
|
+
layout: string;
|
|
87
|
+
kvDtype: string | null;
|
|
88
|
+
layerCount: number;
|
|
89
|
+
digest: string;
|
|
90
|
+
layers: KvCacheLayerByteProof[];
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export declare function digestLogitsForTranscript(
|
|
94
|
+
logits: Float32Array,
|
|
95
|
+
context?: Record<string, unknown> | null
|
|
96
|
+
): ReferenceLogitsDigest;
|
|
97
|
+
|
|
98
|
+
export declare function captureKvCacheByteProof(
|
|
99
|
+
pipeline: Record<string, unknown>,
|
|
100
|
+
enabled: boolean
|
|
101
|
+
): Promise<KvCacheByteProof | null>;
|
|
102
|
+
|
|
26
103
|
export declare function runGeneration(
|
|
27
104
|
pipeline: Record<string, unknown>,
|
|
28
105
|
runtimeConfig: Record<string, unknown>,
|