doppler-gpu 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +67 -34
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.js +5 -1
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/version.js +1 -1
|
@@ -21,8 +21,8 @@ import { isStopToken } from './init.js';
|
|
|
21
21
|
import { embed } from './embed.js';
|
|
22
22
|
import { resolvePerLayerInputsSession } from './generator-helpers.js';
|
|
23
23
|
import { processLayer } from './layer.js';
|
|
24
|
-
import { computeLogits, computeLogitsGPU, recordLogitsGPU, extractLastPositionLogits, finalizeLogits, applySoftcapping } from './logits/index.js';
|
|
25
|
-
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, getWeightDtype } from '../../../gpu/weight-buffer.js';
|
|
24
|
+
import { computeLogits, computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, extractLastPositionLogits, finalizeLogits, applySoftcapping } from './logits/index.js';
|
|
25
|
+
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype, getWeightMetadata } from '../../../gpu/weight-buffer.js';
|
|
26
26
|
import { decodeReadback } from './debug-utils/index.js';
|
|
27
27
|
import { getFinalNormWeights, extractEmbeddingFromHidden } from './generator-runtime.js';
|
|
28
28
|
import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
|
|
@@ -33,10 +33,13 @@ import {
|
|
|
33
33
|
createPleBufferCache,
|
|
34
34
|
prefetchPerLayerRow,
|
|
35
35
|
hasRangeBackedPerLayerInputEmbeddings,
|
|
36
|
+
hasGpuSplitPerLayerInputEmbeddings,
|
|
36
37
|
getPleHotVocabularyRuntime,
|
|
37
38
|
} from './per-layer-inputs.js';
|
|
38
39
|
|
|
39
40
|
const UNKNOWN_TOKEN_TEXT = '<unknown>';
|
|
41
|
+
const FINITENESS_RESET_WORDS = new Uint32Array(4);
|
|
42
|
+
const FINITENESS_STATUS_BYTES = FINITENESS_RESET_WORDS.byteLength;
|
|
40
43
|
|
|
41
44
|
export function sumProfileTimings(timings) {
|
|
42
45
|
if (!timings || Object.keys(timings).length === 0) return null;
|
|
@@ -49,6 +52,23 @@ export function sumProfileTimings(timings) {
|
|
|
49
52
|
return total;
|
|
50
53
|
}
|
|
51
54
|
|
|
55
|
+
function mergeRecordOpLabelCounts(target, source) {
|
|
56
|
+
const merged = target && typeof target === 'object' && !Array.isArray(target)
|
|
57
|
+
? target
|
|
58
|
+
: {};
|
|
59
|
+
if (!source || typeof source !== 'object' || Array.isArray(source)) {
|
|
60
|
+
return merged;
|
|
61
|
+
}
|
|
62
|
+
for (const [label, rawCount] of Object.entries(source)) {
|
|
63
|
+
const count = Number(rawCount);
|
|
64
|
+
if (typeof label !== 'string' || label.length === 0 || !Number.isFinite(count) || count <= 0) {
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
merged[label] = (Number.isFinite(merged[label]) ? merged[label] : 0) + count;
|
|
68
|
+
}
|
|
69
|
+
return merged;
|
|
70
|
+
}
|
|
71
|
+
|
|
52
72
|
function getEffectiveActivationDtype(state, opts) {
|
|
53
73
|
const executionPlanDtype = opts?.executionPlan?.activationDtype;
|
|
54
74
|
if (executionPlanDtype !== undefined && executionPlanDtype !== null) {
|
|
@@ -174,7 +194,53 @@ export function shouldUseFusedDecodeSampling(config) {
|
|
|
174
194
|
&& !hasConvLayers(config.layerTypes ?? []);
|
|
175
195
|
}
|
|
176
196
|
|
|
177
|
-
function
|
|
197
|
+
function shouldUseGreedyLmHeadArgmaxFusion(state, opts, samplingDefaults, repetitionPenalty) {
|
|
198
|
+
const probes = state.runtimeConfig?.shared?.debug?.probes;
|
|
199
|
+
return state.runtimeConfig?.inference?.session?.useGreedyLmHeadArgmaxFusion === true
|
|
200
|
+
&& opts.temperature < samplingDefaults.greedyThreshold
|
|
201
|
+
&& repetitionPenalty === 1.0
|
|
202
|
+
&& state.operatorDiagnostics == null
|
|
203
|
+
&& !(Array.isArray(probes) && probes.length > 0);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
export function createStopTokenLookup(stopTokenIds, eosTokenId) {
|
|
207
|
+
if (!Array.isArray(stopTokenIds)) {
|
|
208
|
+
throw new Error('[Pipeline] stopTokenIds must be an array.');
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const tokenIds = [];
|
|
212
|
+
for (const tokenId of stopTokenIds) {
|
|
213
|
+
if (!tokenIds.includes(tokenId)) {
|
|
214
|
+
tokenIds.push(tokenId);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
if (typeof eosTokenId === 'number' && !tokenIds.includes(eosTokenId)) {
|
|
218
|
+
tokenIds.push(eosTokenId);
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
if (tokenIds.length === 0) {
|
|
222
|
+
return { firstTokenId: null, secondTokenId: null, tokenSet: null };
|
|
223
|
+
}
|
|
224
|
+
if (tokenIds.length === 1) {
|
|
225
|
+
return { firstTokenId: tokenIds[0], secondTokenId: null, tokenSet: null };
|
|
226
|
+
}
|
|
227
|
+
if (tokenIds.length === 2) {
|
|
228
|
+
return { firstTokenId: tokenIds[0], secondTokenId: tokenIds[1], tokenSet: null };
|
|
229
|
+
}
|
|
230
|
+
return { firstTokenId: null, secondTokenId: null, tokenSet: new Set(tokenIds) };
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function lookupHasStopToken(token, stopTokenLookup) {
|
|
234
|
+
if (stopTokenLookup.tokenSet) {
|
|
235
|
+
return stopTokenLookup.tokenSet.has(token);
|
|
236
|
+
}
|
|
237
|
+
if (stopTokenLookup.firstTokenId !== null && token === stopTokenLookup.firstTokenId) {
|
|
238
|
+
return true;
|
|
239
|
+
}
|
|
240
|
+
return stopTokenLookup.secondTokenId !== null && token === stopTokenLookup.secondTokenId;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
export function resolveBatchStop(tokens, stopFlags, stopTokenLookup) {
|
|
178
244
|
let actualCount = tokens.length;
|
|
179
245
|
if (stopFlags) {
|
|
180
246
|
const maxFlags = Math.min(stopFlags.length, tokens.length);
|
|
@@ -187,7 +253,7 @@ function resolveBatchStop(tokens, stopFlags, stopTokenIds, eosTokenId) {
|
|
|
187
253
|
}
|
|
188
254
|
|
|
189
255
|
for (let i = 0; i < actualCount; i++) {
|
|
190
|
-
if (
|
|
256
|
+
if (lookupHasStopToken(tokens[i], stopTokenLookup)) {
|
|
191
257
|
actualCount = i + 1;
|
|
192
258
|
break;
|
|
193
259
|
}
|
|
@@ -215,21 +281,34 @@ export async function readSampledTokenFromStagingBuffer(stagingBuffer, options =
|
|
|
215
281
|
const hasFinitenessBuffer = options.hasFinitenessBuffer === true;
|
|
216
282
|
const ring = options.ring ?? null;
|
|
217
283
|
const cleanupRecorder = options.cleanupRecorder ?? null;
|
|
284
|
+
const timing = {
|
|
285
|
+
mapWaitMs: 0,
|
|
286
|
+
cleanupMs: 0,
|
|
287
|
+
copyMs: 0,
|
|
288
|
+
};
|
|
218
289
|
let mapped = false;
|
|
219
290
|
let cleanupCompleted = false;
|
|
220
291
|
|
|
221
292
|
try {
|
|
293
|
+
const mapStart = performance.now();
|
|
222
294
|
await stagingBuffer.mapAsync(GPUMapMode.READ);
|
|
295
|
+
timing.mapWaitMs = performance.now() - mapStart;
|
|
223
296
|
mapped = true;
|
|
297
|
+
const cleanupStart = performance.now();
|
|
224
298
|
await cleanupRecorder?.completeDeferredCleanup();
|
|
299
|
+
timing.cleanupMs = performance.now() - cleanupStart;
|
|
225
300
|
cleanupCompleted = true;
|
|
301
|
+
const copyStart = performance.now();
|
|
226
302
|
const mappedWords = new Uint32Array(stagingBuffer.getMappedRange());
|
|
227
|
-
|
|
303
|
+
const result = {
|
|
228
304
|
nextToken: mappedWords[0],
|
|
229
305
|
finitenessStatus: hasFinitenessBuffer
|
|
230
306
|
? parseFinitenessStatusWords(mappedWords, 1)
|
|
231
307
|
: parseFinitenessStatusWords(mappedWords, 0),
|
|
308
|
+
timing,
|
|
232
309
|
};
|
|
310
|
+
timing.copyMs = performance.now() - copyStart;
|
|
311
|
+
return result;
|
|
233
312
|
} finally {
|
|
234
313
|
if (mapped) {
|
|
235
314
|
stagingBuffer.unmap();
|
|
@@ -268,6 +347,7 @@ export async function readBatchTokensFromStagingBuffers(options) {
|
|
|
268
347
|
tokensStagingBuffer,
|
|
269
348
|
stopStagingBuffer = null,
|
|
270
349
|
finitenessStagingBuffer = null,
|
|
350
|
+
finitenessOffsetBytes = null,
|
|
271
351
|
tokenCount,
|
|
272
352
|
ownsTokensStaging = false,
|
|
273
353
|
ownsStopStaging = false,
|
|
@@ -279,41 +359,60 @@ export async function readBatchTokensFromStagingBuffers(options) {
|
|
|
279
359
|
let stopMapped = false;
|
|
280
360
|
let finitenessMapped = false;
|
|
281
361
|
let cleanupCompleted = false;
|
|
362
|
+
const hasPackedFiniteness = Number.isFinite(finitenessOffsetBytes) && finitenessOffsetBytes >= 0;
|
|
363
|
+
const timing = {
|
|
364
|
+
mapWaitMs: 0,
|
|
365
|
+
cleanupMs: 0,
|
|
366
|
+
copyMs: 0,
|
|
367
|
+
};
|
|
282
368
|
|
|
283
369
|
try {
|
|
370
|
+
const mapStart = performance.now();
|
|
284
371
|
const mapPromises = [tokensStagingBuffer.mapAsync(GPUMapMode.READ)];
|
|
285
372
|
if (stopStagingBuffer) {
|
|
286
373
|
mapPromises.push(stopStagingBuffer.mapAsync(GPUMapMode.READ));
|
|
287
374
|
}
|
|
288
|
-
if (finitenessStagingBuffer) {
|
|
375
|
+
if (finitenessStagingBuffer && !hasPackedFiniteness) {
|
|
289
376
|
mapPromises.push(finitenessStagingBuffer.mapAsync(GPUMapMode.READ));
|
|
290
377
|
}
|
|
291
378
|
const mapResults = await Promise.allSettled(mapPromises);
|
|
379
|
+
timing.mapWaitMs = performance.now() - mapStart;
|
|
292
380
|
tokensMapped = mapResults[0]?.status === 'fulfilled';
|
|
293
381
|
stopMapped = Boolean(stopStagingBuffer) && mapResults[1]?.status === 'fulfilled';
|
|
294
|
-
finitenessMapped = Boolean(finitenessStagingBuffer)
|
|
382
|
+
finitenessMapped = Boolean(finitenessStagingBuffer) && !hasPackedFiniteness
|
|
295
383
|
&& mapResults[stopStagingBuffer ? 2 : 1]?.status === 'fulfilled';
|
|
296
384
|
const mapFailure = mapResults.find((result) => result.status === 'rejected');
|
|
297
385
|
if (mapFailure) {
|
|
298
386
|
throw mapFailure.reason;
|
|
299
387
|
}
|
|
388
|
+
const cleanupStart = performance.now();
|
|
300
389
|
await cleanupRecorder?.completeDeferredCleanup();
|
|
390
|
+
timing.cleanupMs = performance.now() - cleanupStart;
|
|
301
391
|
cleanupCompleted = true;
|
|
302
392
|
|
|
303
|
-
const
|
|
304
|
-
|
|
305
|
-
);
|
|
306
|
-
const
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
393
|
+
const copyStart = performance.now();
|
|
394
|
+
const tokensRange = tokensStagingBuffer.getMappedRange();
|
|
395
|
+
const tokenWords = new Uint32Array(tokensRange, 0, tokenCount);
|
|
396
|
+
const tokens = new Uint32Array(tokenWords.length);
|
|
397
|
+
tokens.set(tokenWords);
|
|
398
|
+
let stopFlags = null;
|
|
399
|
+
if (stopStagingBuffer) {
|
|
400
|
+
const stopWords = new Uint32Array(stopStagingBuffer.getMappedRange()).subarray(0, tokenCount);
|
|
401
|
+
stopFlags = new Uint32Array(stopWords.length);
|
|
402
|
+
stopFlags.set(stopWords);
|
|
403
|
+
}
|
|
404
|
+
const finitenessStatus = hasPackedFiniteness
|
|
405
|
+
? parseFinitenessStatusWords(new Uint32Array(tokensRange, finitenessOffsetBytes, 4), 0)
|
|
406
|
+
: finitenessStagingBuffer
|
|
310
407
|
? parseFinitenessStatusWords(new Uint32Array(finitenessStagingBuffer.getMappedRange()), 0)
|
|
311
408
|
: { triggered: false, metadata: '' };
|
|
409
|
+
timing.copyMs = performance.now() - copyStart;
|
|
312
410
|
|
|
313
411
|
return {
|
|
314
412
|
tokens,
|
|
315
413
|
stopFlags,
|
|
316
414
|
finitenessStatus,
|
|
415
|
+
timing,
|
|
317
416
|
};
|
|
318
417
|
} finally {
|
|
319
418
|
if (finitenessMapped) {
|
|
@@ -354,13 +453,14 @@ async function runDecodeLayers(state, tokenId, opts, helpers) {
|
|
|
354
453
|
const decodeAltBuffer = state.decodeBuffers.getOutputHiddenBuffer();
|
|
355
454
|
|
|
356
455
|
const embedBufferRaw = state.weights.get('embed');
|
|
357
|
-
if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
|
|
456
|
+
if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
|
|
358
457
|
throw new Error('Embed buffer not found or not a supported buffer type');
|
|
359
458
|
}
|
|
360
459
|
const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
|
|
361
460
|
const embedDtype = isCpuWeightBuffer(embedBufferRaw)
|
|
362
461
|
? embedBufferRaw.dtype
|
|
363
462
|
: getWeightDtype(embedBufferRaw);
|
|
463
|
+
const embedMetadata = getWeightMetadata(embedBufferRaw);
|
|
364
464
|
const activationDtype = getEffectiveActivationDtype(state, opts);
|
|
365
465
|
|
|
366
466
|
const embedTensor = await embed([tokenId], embedBuffer, {
|
|
@@ -372,7 +472,8 @@ async function runDecodeLayers(state, tokenId, opts, helpers) {
|
|
|
372
472
|
debugProbes: state.runtimeConfig.shared.debug.probes,
|
|
373
473
|
operatorDiagnostics: state.operatorDiagnostics,
|
|
374
474
|
activationDtype,
|
|
375
|
-
embeddingDtype: selectRuleValue('inference', 'dtype', '
|
|
475
|
+
embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
|
|
476
|
+
embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
|
|
376
477
|
executionPolicies: state.executionV1State?.policies ?? null,
|
|
377
478
|
});
|
|
378
479
|
|
|
@@ -437,8 +538,8 @@ function createDecodeRecorder(state, opts) {
|
|
|
437
538
|
let recorder;
|
|
438
539
|
if (recorderEnabled) {
|
|
439
540
|
recorder = opts.profile
|
|
440
|
-
? createProfilingRecorder('decode')
|
|
441
|
-
: createCommandRecorder('decode');
|
|
541
|
+
? createProfilingRecorder('decode', device)
|
|
542
|
+
: createCommandRecorder('decode', { recordLabels: opts.debug === true }, device);
|
|
442
543
|
}
|
|
443
544
|
if (state.decodeStepCount === 1) {
|
|
444
545
|
const path = selectRuleValue('inference', 'config', 'tracePath', { useRecorder: Boolean(recorder) });
|
|
@@ -491,7 +592,7 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
491
592
|
const recorder = createDecodeRecorder(state, opts);
|
|
492
593
|
|
|
493
594
|
if (state.finitenessBuffer && device) {
|
|
494
|
-
device.queue.writeBuffer(state.finitenessBuffer, 0,
|
|
595
|
+
device.queue.writeBuffer(state.finitenessBuffer, 0, FINITENESS_RESET_WORDS);
|
|
495
596
|
}
|
|
496
597
|
|
|
497
598
|
const context = helpers.buildLayerContext(recorder, true, opts.debugLayers, executionPlan);
|
|
@@ -503,13 +604,14 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
503
604
|
const decodeAltBuffer = state.decodeBuffers.getOutputHiddenBuffer();
|
|
504
605
|
|
|
505
606
|
const embedBufferRaw = state.weights.get('embed');
|
|
506
|
-
if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
|
|
607
|
+
if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
|
|
507
608
|
throw new Error('Embed buffer not found or not a supported buffer type');
|
|
508
609
|
}
|
|
509
610
|
const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
|
|
510
611
|
const embedDtype = isCpuWeightBuffer(embedBufferRaw)
|
|
511
612
|
? embedBufferRaw.dtype
|
|
512
613
|
: getWeightDtype(embedBufferRaw);
|
|
614
|
+
const embedMetadata = getWeightMetadata(embedBufferRaw);
|
|
513
615
|
const activationDtype = getEffectiveActivationDtype(state, opts);
|
|
514
616
|
const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
|
|
515
617
|
|
|
@@ -523,7 +625,8 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
523
625
|
debugProbes: state.runtimeConfig.shared.debug.probes,
|
|
524
626
|
operatorDiagnostics: state.operatorDiagnostics,
|
|
525
627
|
activationDtype,
|
|
526
|
-
embeddingDtype: selectRuleValue('inference', 'dtype', '
|
|
628
|
+
embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
|
|
629
|
+
embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
|
|
527
630
|
executionPolicies: state.executionV1State?.policies ?? null,
|
|
528
631
|
});
|
|
529
632
|
|
|
@@ -708,6 +811,14 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
708
811
|
|
|
709
812
|
state.stats.singleTokenSubmitWaitMs = (state.stats.singleTokenSubmitWaitMs ?? 0) + submitWaitMs;
|
|
710
813
|
state.stats.singleTokenReadbackWaitMs = (state.stats.singleTokenReadbackWaitMs ?? 0) + readbackWaitMs;
|
|
814
|
+
if (readbackResult.timing) {
|
|
815
|
+
state.stats.singleTokenReadbackMapWaitMs = (state.stats.singleTokenReadbackMapWaitMs ?? 0)
|
|
816
|
+
+ readbackResult.timing.mapWaitMs;
|
|
817
|
+
state.stats.singleTokenReadbackCleanupMs = (state.stats.singleTokenReadbackCleanupMs ?? 0)
|
|
818
|
+
+ readbackResult.timing.cleanupMs;
|
|
819
|
+
state.stats.singleTokenReadbackCopyMs = (state.stats.singleTokenReadbackCopyMs ?? 0)
|
|
820
|
+
+ readbackResult.timing.copyMs;
|
|
821
|
+
}
|
|
711
822
|
|
|
712
823
|
const { nextToken: fusedNextToken, finitenessStatus } = readbackResult;
|
|
713
824
|
|
|
@@ -1209,8 +1320,8 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1209
1320
|
state.batchingStats.batchedForwardCalls += 1;
|
|
1210
1321
|
const tokensPerInterval = batchSize * readbackInterval;
|
|
1211
1322
|
const recorder = opts.profile
|
|
1212
|
-
? createProfilingRecorder('batch_decode')
|
|
1213
|
-
: createCommandRecorder('batch_decode');
|
|
1323
|
+
? createProfilingRecorder('batch_decode', device)
|
|
1324
|
+
: createCommandRecorder('batch_decode', { recordLabels: opts.debug === true }, device);
|
|
1214
1325
|
const lmHead = state.weights.get('lm_head');
|
|
1215
1326
|
if (lmHead && isCpuWeightBuffer(lmHead)) {
|
|
1216
1327
|
throw new Error('[Pipeline] GPU-only decode not supported with CPU-resident LM head.');
|
|
@@ -1227,6 +1338,10 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1227
1338
|
config,
|
|
1228
1339
|
weights: state.weights,
|
|
1229
1340
|
});
|
|
1341
|
+
const hasGpuSplitPerLayerInputs = hasGpuSplitPerLayerInputEmbeddings({
|
|
1342
|
+
config,
|
|
1343
|
+
weights: state.weights,
|
|
1344
|
+
});
|
|
1230
1345
|
const pleHotVocabularyRuntime = getPleHotVocabularyRuntime({ weights: state.weights });
|
|
1231
1346
|
const hotStartTokenIndex = pleHotVocabularyRuntime?.hotTokenIndexMap?.[startToken] ?? null;
|
|
1232
1347
|
const canUseHotVocabularyBatchDecode = hasRangeBackedPerLayerInputs
|
|
@@ -1260,6 +1375,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1260
1375
|
if (eosTokenId == null) {
|
|
1261
1376
|
throw new Error('[Pipeline] Missing EOS token. Ensure tokenizer or manifest provides stop tokens.');
|
|
1262
1377
|
}
|
|
1378
|
+
const stopTokenLookup = createStopTokenLookup(stopTokenIds, eosToken);
|
|
1263
1379
|
const maxTokens = executionPlan?.maxTokens
|
|
1264
1380
|
?? opts.maxTokens
|
|
1265
1381
|
?? state.runtimeConfig.inference.generation.maxTokens;
|
|
@@ -1304,8 +1420,10 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1304
1420
|
})
|
|
1305
1421
|
: null;
|
|
1306
1422
|
|
|
1423
|
+
const tokenReadbackBytes = N * 4;
|
|
1424
|
+
const tokenStagingBytes = tokenReadbackBytes + (state.finitenessBuffer ? FINITENESS_STATUS_BYTES : 0);
|
|
1307
1425
|
const tokensStagingBuffer = ringSlot?.stagingTokens ?? device.createBuffer({
|
|
1308
|
-
size:
|
|
1426
|
+
size: tokenStagingBytes,
|
|
1309
1427
|
usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
|
|
1310
1428
|
});
|
|
1311
1429
|
const ownsTokensStaging = !ringSlot?.stagingTokens;
|
|
@@ -1317,24 +1435,33 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1317
1435
|
})
|
|
1318
1436
|
: null;
|
|
1319
1437
|
const ownsStopStaging = useGpuStopFlags && !ringSlot?.stagingStop;
|
|
1320
|
-
const
|
|
1438
|
+
const packedFinitenessOffsetBytes = state.finitenessBuffer
|
|
1439
|
+
&& tokensStagingBuffer.size >= tokenReadbackBytes + FINITENESS_STATUS_BYTES
|
|
1440
|
+
? tokenReadbackBytes
|
|
1441
|
+
: null;
|
|
1442
|
+
const finitenessStagingBuffer = state.finitenessBuffer && packedFinitenessOffsetBytes == null
|
|
1321
1443
|
? ringSlot?.stagingFiniteness ?? device.createBuffer({
|
|
1322
|
-
size:
|
|
1444
|
+
size: FINITENESS_STATUS_BYTES,
|
|
1323
1445
|
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
|
1324
1446
|
})
|
|
1325
1447
|
: null;
|
|
1326
|
-
const ownsFinitenessStaging = Boolean(state.finitenessBuffer)
|
|
1448
|
+
const ownsFinitenessStaging = Boolean(state.finitenessBuffer)
|
|
1449
|
+
&& packedFinitenessOffsetBytes == null
|
|
1450
|
+
&& !ringSlot?.stagingFiniteness;
|
|
1327
1451
|
let readbackCleanupDelegated = false;
|
|
1328
1452
|
let repHistoryBuffer = null;
|
|
1329
1453
|
let repHistoryCount = 0;
|
|
1330
1454
|
try {
|
|
1331
1455
|
if (state.finitenessBuffer) {
|
|
1332
|
-
device.queue.writeBuffer(state.finitenessBuffer, 0,
|
|
1456
|
+
device.queue.writeBuffer(state.finitenessBuffer, 0, FINITENESS_RESET_WORDS);
|
|
1333
1457
|
}
|
|
1334
1458
|
|
|
1335
|
-
|
|
1459
|
+
const singleWordUpload = new Uint32Array(1);
|
|
1460
|
+
singleWordUpload[0] = startToken;
|
|
1461
|
+
device.queue.writeBuffer(tokensBuffer, 0, singleWordUpload);
|
|
1336
1462
|
if (pleInputTokensBuffer) {
|
|
1337
|
-
|
|
1463
|
+
singleWordUpload[0] = hotStartTokenIndex;
|
|
1464
|
+
device.queue.writeBuffer(pleInputTokensBuffer, 0, singleWordUpload);
|
|
1338
1465
|
}
|
|
1339
1466
|
if (stopBuffer) {
|
|
1340
1467
|
const stopElements = stopBuffer.size / 4;
|
|
@@ -1350,11 +1477,12 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1350
1477
|
if (isCpuWeightBuffer(embedBufferRaw)) {
|
|
1351
1478
|
throw new Error('[Pipeline] GPU-only decode not supported with CPU-resident embeddings.');
|
|
1352
1479
|
}
|
|
1353
|
-
if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw)) {
|
|
1480
|
+
if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw)) {
|
|
1354
1481
|
throw new Error('Embed buffer not found or not a GPUBuffer/WeightBuffer');
|
|
1355
1482
|
}
|
|
1356
1483
|
const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
|
|
1357
1484
|
const embedDtype = getWeightDtype(embedBufferRaw);
|
|
1485
|
+
const embedMetadata = getWeightMetadata(embedBufferRaw);
|
|
1358
1486
|
const activationDtype = getEffectiveActivationDtype(state, opts);
|
|
1359
1487
|
|
|
1360
1488
|
// GPU-side repetition penalty: upload deduplicated history before batch
|
|
@@ -1373,9 +1501,15 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1373
1501
|
}
|
|
1374
1502
|
|
|
1375
1503
|
// Hoist loop-invariant values to avoid repeated rule lookups and allocations.
|
|
1376
|
-
const embeddingDtype = selectRuleValue('inference', 'dtype', '
|
|
1504
|
+
const embeddingDtype = selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype });
|
|
1377
1505
|
const debugProbes = state.runtimeConfig.shared.debug.probes;
|
|
1378
1506
|
const currentTokenIdsArray = [startToken];
|
|
1507
|
+
const useGreedyLmHeadArgmaxFusion = shouldUseGreedyLmHeadArgmaxFusion(
|
|
1508
|
+
state,
|
|
1509
|
+
opts,
|
|
1510
|
+
samplingDefaults,
|
|
1511
|
+
repetitionPenalty
|
|
1512
|
+
);
|
|
1379
1513
|
|
|
1380
1514
|
for (let i = 0; i < N; i++) {
|
|
1381
1515
|
// In the GPU batch path, only the start token (i=0) is known on the CPU.
|
|
@@ -1394,11 +1528,13 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1394
1528
|
vocabSize: config.vocabSize,
|
|
1395
1529
|
scaleEmbeddings: config.scaleEmbeddings,
|
|
1396
1530
|
recorder,
|
|
1531
|
+
outputBuffer: context.decodeBuffers?.getHiddenBuffer() ?? undefined,
|
|
1397
1532
|
transpose: state.embeddingTranspose,
|
|
1398
1533
|
debugProbes,
|
|
1399
1534
|
operatorDiagnostics: state.operatorDiagnostics,
|
|
1400
1535
|
activationDtype,
|
|
1401
1536
|
embeddingDtype,
|
|
1537
|
+
embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
|
|
1402
1538
|
executionPolicies: state.executionV1State?.policies ?? null,
|
|
1403
1539
|
numTokens: 1,
|
|
1404
1540
|
indexOffset: i,
|
|
@@ -1408,7 +1544,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1408
1544
|
const perLayerInputs = await preparePerLayerInputs(tokensBuffer, hiddenTensor, context, {
|
|
1409
1545
|
numTokens: 1,
|
|
1410
1546
|
indexOffset: i,
|
|
1411
|
-
perLayerTokenIds: pleInputTokensBuffer,
|
|
1547
|
+
perLayerTokenIds: pleInputTokensBuffer ?? (hasGpuSplitPerLayerInputs ? tokensBuffer : null),
|
|
1412
1548
|
perLayerIndexOffset: i,
|
|
1413
1549
|
tokenIdHint: i === 0 ? startToken : null,
|
|
1414
1550
|
pleCache: state.pleCache ?? null,
|
|
@@ -1451,48 +1587,67 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1451
1587
|
helpers.releaseSharedAttentionState?.(context.sharedAttentionState, recorder);
|
|
1452
1588
|
}
|
|
1453
1589
|
|
|
1454
|
-
const logits = await recordLogitsGPU(
|
|
1455
|
-
recorder,
|
|
1456
|
-
hiddenStatesBuffer,
|
|
1457
|
-
1,
|
|
1458
|
-
helpers.getLogitsWeights(),
|
|
1459
|
-
helpers.getLogitsConfig(),
|
|
1460
|
-
state.operatorDiagnostics
|
|
1461
|
-
);
|
|
1462
|
-
const { logitsBuffer, vocabSize, logitsDtype } = logits;
|
|
1463
|
-
|
|
1464
|
-
// Apply GPU-side repetition penalty before sampling
|
|
1465
|
-
if (repHistoryBuffer && repetitionPenalty !== 1.0) {
|
|
1466
|
-
await recordRepPenalty(recorder, logitsBuffer, repHistoryBuffer, tokensBuffer, {
|
|
1467
|
-
vocabSize,
|
|
1468
|
-
historyCount: repHistoryCount,
|
|
1469
|
-
penalty: repetitionPenalty,
|
|
1470
|
-
batchCount: i,
|
|
1471
|
-
batchOffset: 1,
|
|
1472
|
-
logitsDtype,
|
|
1473
|
-
});
|
|
1474
|
-
}
|
|
1475
|
-
|
|
1476
1590
|
const outputIndex = i + 1;
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1591
|
+
let logitsBuffer = null;
|
|
1592
|
+
if (useGreedyLmHeadArgmaxFusion) {
|
|
1593
|
+
await recordGreedyLmHeadArgmaxGPU(
|
|
1594
|
+
recorder,
|
|
1595
|
+
hiddenStatesBuffer,
|
|
1596
|
+
1,
|
|
1597
|
+
helpers.getLogitsWeights(),
|
|
1598
|
+
helpers.getLogitsConfig(),
|
|
1599
|
+
{
|
|
1600
|
+
padTokenId,
|
|
1601
|
+
logitSoftcap,
|
|
1602
|
+
outputBuffer: tokensBuffer,
|
|
1603
|
+
outputIndex,
|
|
1604
|
+
},
|
|
1605
|
+
state.operatorDiagnostics
|
|
1606
|
+
);
|
|
1485
1607
|
} else {
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1608
|
+
const logits = await recordLogitsGPU(
|
|
1609
|
+
recorder,
|
|
1610
|
+
hiddenStatesBuffer,
|
|
1611
|
+
1,
|
|
1612
|
+
helpers.getLogitsWeights(),
|
|
1613
|
+
helpers.getLogitsConfig(),
|
|
1614
|
+
state.operatorDiagnostics
|
|
1615
|
+
);
|
|
1616
|
+
const { vocabSize, logitsDtype } = logits;
|
|
1617
|
+
logitsBuffer = logits.logitsBuffer;
|
|
1618
|
+
|
|
1619
|
+
// Apply GPU-side repetition penalty before sampling
|
|
1620
|
+
if (repHistoryBuffer && repetitionPenalty !== 1.0) {
|
|
1621
|
+
await recordRepPenalty(recorder, logitsBuffer, repHistoryBuffer, tokensBuffer, {
|
|
1622
|
+
vocabSize,
|
|
1623
|
+
historyCount: repHistoryCount,
|
|
1624
|
+
penalty: repetitionPenalty,
|
|
1625
|
+
batchCount: i,
|
|
1626
|
+
batchOffset: 1,
|
|
1627
|
+
logitsDtype,
|
|
1628
|
+
});
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1631
|
+
if (opts.temperature < samplingDefaults.greedyThreshold) {
|
|
1632
|
+
await recordArgmax(recorder, logitsBuffer, vocabSize, {
|
|
1633
|
+
padTokenId,
|
|
1634
|
+
logitSoftcap,
|
|
1635
|
+
logitsDtype,
|
|
1636
|
+
outputBuffer: tokensBuffer,
|
|
1637
|
+
outputIndex,
|
|
1638
|
+
});
|
|
1639
|
+
} else {
|
|
1640
|
+
await recordGPUSample(recorder, logitsBuffer, vocabSize, {
|
|
1641
|
+
temperature: opts.temperature,
|
|
1642
|
+
topK: opts.topK,
|
|
1643
|
+
padTokenId,
|
|
1644
|
+
logitSoftcap,
|
|
1645
|
+
logitsDtype,
|
|
1646
|
+
outputBuffer: tokensBuffer,
|
|
1647
|
+
outputIndex,
|
|
1648
|
+
greedyThreshold: samplingDefaults.greedyThreshold,
|
|
1649
|
+
});
|
|
1650
|
+
}
|
|
1496
1651
|
}
|
|
1497
1652
|
|
|
1498
1653
|
const stopCheck = canUseHotVocabularyBatchDecode
|
|
@@ -1530,7 +1685,16 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1530
1685
|
}
|
|
1531
1686
|
|
|
1532
1687
|
const recordMs = performance.now() - recordStart;
|
|
1688
|
+
const recordStats = recorder.getStats();
|
|
1689
|
+
const recordOps = recordStats.opCount;
|
|
1690
|
+
const recordPasses = recordStats.computePassCount;
|
|
1533
1691
|
state.stats.decodeRecordMs = (state.stats.decodeRecordMs ?? 0) + recordMs;
|
|
1692
|
+
state.stats.decodeRecordOps = (state.stats.decodeRecordOps ?? 0) + recordOps;
|
|
1693
|
+
state.stats.decodeRecordPasses = (state.stats.decodeRecordPasses ?? 0) + recordPasses;
|
|
1694
|
+
state.stats.decodeRecordOpLabels = mergeRecordOpLabelCounts(
|
|
1695
|
+
state.stats.decodeRecordOpLabels,
|
|
1696
|
+
recordStats.opLabelCounts
|
|
1697
|
+
);
|
|
1534
1698
|
|
|
1535
1699
|
const encoder = recorder.getEncoder();
|
|
1536
1700
|
encoder.copyBufferToBuffer(tokensBuffer, 4, tokensStagingBuffer, 0, N * 4);
|
|
@@ -1538,8 +1702,22 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1538
1702
|
encoder.copyBufferToBuffer(stopBuffer, 4, stopStagingBuffer, 0, N * 4);
|
|
1539
1703
|
}
|
|
1540
1704
|
|
|
1541
|
-
if (state.finitenessBuffer &&
|
|
1542
|
-
encoder.copyBufferToBuffer(
|
|
1705
|
+
if (state.finitenessBuffer && packedFinitenessOffsetBytes != null) {
|
|
1706
|
+
encoder.copyBufferToBuffer(
|
|
1707
|
+
state.finitenessBuffer,
|
|
1708
|
+
0,
|
|
1709
|
+
tokensStagingBuffer,
|
|
1710
|
+
packedFinitenessOffsetBytes,
|
|
1711
|
+
FINITENESS_STATUS_BYTES
|
|
1712
|
+
);
|
|
1713
|
+
} else if (state.finitenessBuffer && finitenessStagingBuffer) {
|
|
1714
|
+
encoder.copyBufferToBuffer(
|
|
1715
|
+
state.finitenessBuffer,
|
|
1716
|
+
0,
|
|
1717
|
+
finitenessStagingBuffer,
|
|
1718
|
+
0,
|
|
1719
|
+
FINITENESS_STATUS_BYTES
|
|
1720
|
+
);
|
|
1543
1721
|
}
|
|
1544
1722
|
|
|
1545
1723
|
if (!allowReadback('pipeline.decode.sample')) {
|
|
@@ -1554,6 +1732,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1554
1732
|
tokensStagingBuffer,
|
|
1555
1733
|
stopStagingBuffer,
|
|
1556
1734
|
finitenessStagingBuffer,
|
|
1735
|
+
finitenessOffsetBytes: packedFinitenessOffsetBytes,
|
|
1557
1736
|
tokenCount: N,
|
|
1558
1737
|
ownsTokensStaging,
|
|
1559
1738
|
ownsStopStaging,
|
|
@@ -1563,6 +1742,14 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1563
1742
|
});
|
|
1564
1743
|
const readbackWaitMs = performance.now() - readbackStart;
|
|
1565
1744
|
state.stats.decodeReadbackWaitMs = (state.stats.decodeReadbackWaitMs ?? 0) + readbackWaitMs;
|
|
1745
|
+
if (readback.timing) {
|
|
1746
|
+
state.stats.decodeReadbackMapWaitMs = (state.stats.decodeReadbackMapWaitMs ?? 0)
|
|
1747
|
+
+ readback.timing.mapWaitMs;
|
|
1748
|
+
state.stats.decodeReadbackCleanupMs = (state.stats.decodeReadbackCleanupMs ?? 0)
|
|
1749
|
+
+ readback.timing.cleanupMs;
|
|
1750
|
+
state.stats.decodeReadbackCopyMs = (state.stats.decodeReadbackCopyMs ?? 0)
|
|
1751
|
+
+ readback.timing.copyMs;
|
|
1752
|
+
}
|
|
1566
1753
|
|
|
1567
1754
|
const isInfinite = readback.finitenessStatus.triggered;
|
|
1568
1755
|
const metadata = readback.finitenessStatus.metadata;
|
|
@@ -1581,8 +1768,10 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1581
1768
|
log.debug('Pipeline', `[STOP] N=${N} flags=[${Array.from(stopFlags).join(',')}] tokens=[${tokens.join(',')}] eos=${eosTokenId}`);
|
|
1582
1769
|
}
|
|
1583
1770
|
|
|
1584
|
-
const actualCount = resolveBatchStop(tokens, stopFlags,
|
|
1585
|
-
const generatedTokens = tokens.
|
|
1771
|
+
const actualCount = resolveBatchStop(tokens, stopFlags, stopTokenLookup);
|
|
1772
|
+
const generatedTokens = actualCount === tokens.length
|
|
1773
|
+
? tokens
|
|
1774
|
+
: tokens.subarray(0, actualCount);
|
|
1586
1775
|
const invalidToken = findInvalidGeneratedToken(generatedTokens, config.vocabSize, padTokenId);
|
|
1587
1776
|
|
|
1588
1777
|
if (isInfinite) {
|
|
@@ -1595,6 +1784,8 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1595
1784
|
`at batch index ${invalidToken.index} (vocabSize=${config.vocabSize}, padTokenId=${padTokenId ?? 'none'}).`
|
|
1596
1785
|
);
|
|
1597
1786
|
}
|
|
1787
|
+
state.batchingStats.executedBatchTokens = (state.batchingStats.executedBatchTokens ?? 0) + N;
|
|
1788
|
+
state.batchingStats.resolvedBatchTokens = (state.batchingStats.resolvedBatchTokens ?? 0) + actualCount;
|
|
1598
1789
|
|
|
1599
1790
|
if (opts.profile && recorder.isProfilingEnabled()) {
|
|
1600
1791
|
const timings = await recorder.resolveProfileTimings();
|