doppler-gpu 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +67 -34
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.js +5 -1
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/version.js +1 -1
|
@@ -5,6 +5,7 @@ import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js'
|
|
|
5
5
|
import { runMatmul, runRMSNorm } from '../../../../gpu/kernel-selector.js';
|
|
6
6
|
import { recordMatmul } from '../../../../gpu/kernels/matmul.js';
|
|
7
7
|
import { recordRMSNorm } from '../../../../gpu/kernels/rmsnorm.js';
|
|
8
|
+
import { recordLmHeadArgmaxF16 } from '../../../../gpu/kernels/lm-head-argmax.js';
|
|
8
9
|
import { createTensor } from '../../../../gpu/tensor.js';
|
|
9
10
|
import {
|
|
10
11
|
castF16ToF32,
|
|
@@ -12,7 +13,7 @@ import {
|
|
|
12
13
|
recordCastF16ToF32,
|
|
13
14
|
recordCastF32ToF16,
|
|
14
15
|
} from '../../../../gpu/kernels/cast.js';
|
|
15
|
-
import { createWeightBuffer, isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance } from '../../../../gpu/weight-buffer.js';
|
|
16
|
+
import { createWeightBuffer, createSplitWeightBuffer, isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer } from '../../../../gpu/weight-buffer.js';
|
|
16
17
|
import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
|
|
17
18
|
import { getRuntimeConfig } from '../../../../config/runtime.js';
|
|
18
19
|
import { getKernelPathMatmulPrecision, getKernelPathStepPrecision } from '../../../../config/kernel-path-loader.js';
|
|
@@ -58,6 +59,10 @@ function resolvePostLayerStepDtype(op, phase, kernelPath, fallback, field) {
|
|
|
58
59
|
return resolvePrecisionFieldDtype(precision, fallback, field);
|
|
59
60
|
}
|
|
60
61
|
|
|
62
|
+
function resolveLmHeadMatmulRole(phase) {
|
|
63
|
+
return phase === 'prefill' ? 'lm_head_prefill' : 'lm_head';
|
|
64
|
+
}
|
|
65
|
+
|
|
61
66
|
async function coerceTensorDtype(tensor, targetDtype, recorder = null, options = {}) {
|
|
62
67
|
if (!targetDtype || tensor.dtype === targetDtype) {
|
|
63
68
|
return tensor;
|
|
@@ -143,6 +148,7 @@ function createStableF32LogitsKernelPath(kernelPath) {
|
|
|
143
148
|
|
|
144
149
|
const bf16ScratchU32 = new Uint32Array(1);
|
|
145
150
|
const bf16ScratchF32 = new Float32Array(bf16ScratchU32.buffer);
|
|
151
|
+
const SPLIT_UPLOAD_CHUNK_BYTES = 64 * 1024 * 1024;
|
|
146
152
|
|
|
147
153
|
function bf16ToF32(value) {
|
|
148
154
|
bf16ScratchU32[0] = (value & 0xffff) << 16;
|
|
@@ -158,6 +164,17 @@ function isRangeBackedCpuWeightSource(value) {
|
|
|
158
164
|
);
|
|
159
165
|
}
|
|
160
166
|
|
|
167
|
+
function alignByteLength(byteLength) {
|
|
168
|
+
return Math.ceil(byteLength / 4) * 4;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function writeBufferInChunks(queue, buffer, bytes) {
|
|
172
|
+
for (let offset = 0; offset < bytes.byteLength; offset += SPLIT_UPLOAD_CHUNK_BYTES) {
|
|
173
|
+
const end = Math.min(offset + SPLIT_UPLOAD_CHUNK_BYTES, bytes.byteLength);
|
|
174
|
+
queue.writeBuffer(buffer, offset, bytes, offset, end - offset);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
161
178
|
function normalizeRangeBytes(value, label) {
|
|
162
179
|
if (value instanceof Uint8Array) return value;
|
|
163
180
|
if (value instanceof ArrayBuffer) return new Uint8Array(value);
|
|
@@ -357,6 +374,141 @@ export function writeChunkLogits(
|
|
|
357
374
|
}
|
|
358
375
|
|
|
359
376
|
|
|
377
|
+
function resolveSplitLmHeadRows(device, hiddenSize, largeWeightConfig) {
|
|
378
|
+
if (largeWeightConfig.safetyRatio == null) {
|
|
379
|
+
throw new Error('runtime.inference.largeWeights.safetyRatio is required.');
|
|
380
|
+
}
|
|
381
|
+
const safety = Math.min(Math.max(largeWeightConfig.safetyRatio, 0.1), 1);
|
|
382
|
+
const maxBinding = Math.min(device.limits.maxStorageBufferBindingSize, device.limits.maxBufferSize);
|
|
383
|
+
const maxBytes = Math.floor(maxBinding * safety);
|
|
384
|
+
const rowBytes = hiddenSize * 2;
|
|
385
|
+
const rowsByBinding = Math.floor(maxBytes / rowBytes);
|
|
386
|
+
const requested = largeWeightConfig.lmHeadChunkRows;
|
|
387
|
+
const rows = Number.isInteger(requested) && requested > 0
|
|
388
|
+
? Math.min(requested, rowsByBinding)
|
|
389
|
+
: rowsByBinding;
|
|
390
|
+
if (!Number.isFinite(rows) || rows <= 0) {
|
|
391
|
+
throw new Error(
|
|
392
|
+
`[Logits] split LM head row size underflow (maxBytes=${maxBytes}, hiddenSize=${hiddenSize}).`
|
|
393
|
+
);
|
|
394
|
+
}
|
|
395
|
+
return rows;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
export function shouldMaterializeSplitLmHeadGPU(lmHead, largeWeightConfig) {
|
|
399
|
+
const overrides = largeWeightConfig?.gpuResidentOverrides;
|
|
400
|
+
if (!Array.isArray(overrides) || overrides.length === 0) {
|
|
401
|
+
return false;
|
|
402
|
+
}
|
|
403
|
+
const label = lmHead?.label;
|
|
404
|
+
return typeof label === 'string' && overrides.includes(label);
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
function destroySplitWeightBuffer(splitWeight) {
|
|
408
|
+
if (!splitWeight) {
|
|
409
|
+
return;
|
|
410
|
+
}
|
|
411
|
+
for (const section of splitWeight.sections ?? []) {
|
|
412
|
+
try {
|
|
413
|
+
section.buffer.destroy();
|
|
414
|
+
} catch {}
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
async function readSplitLmHeadSectionBytes(lmHead, hiddenSize, rowStart, rowCount) {
|
|
419
|
+
const byteOffset = rowStart * hiddenSize * 2;
|
|
420
|
+
const byteLength = rowCount * hiddenSize * 2;
|
|
421
|
+
const data = lmHead.data;
|
|
422
|
+
if (isRangeBackedCpuWeightSource(data)) {
|
|
423
|
+
const bytes = normalizeRangeBytes(
|
|
424
|
+
await data.loadRange(byteOffset, byteLength),
|
|
425
|
+
'CPU LM head split range source'
|
|
426
|
+
);
|
|
427
|
+
if (bytes.byteLength !== byteLength) {
|
|
428
|
+
throw new Error(
|
|
429
|
+
`[Logits] CPU LM head split source returned ${bytes.byteLength} bytes, expected ${byteLength}.`
|
|
430
|
+
);
|
|
431
|
+
}
|
|
432
|
+
return bytes;
|
|
433
|
+
}
|
|
434
|
+
if (data instanceof Uint16Array) {
|
|
435
|
+
return new Uint8Array(data.buffer, data.byteOffset + byteOffset, byteLength);
|
|
436
|
+
}
|
|
437
|
+
return null;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
async function materializeSplitLmHeadGPU(lmHead, hiddenSize, weightVocabSize, largeWeightConfig) {
|
|
441
|
+
if (!shouldMaterializeSplitLmHeadGPU(lmHead, largeWeightConfig)) {
|
|
442
|
+
if (lmHead.gpuSplitWeight) {
|
|
443
|
+
destroySplitWeightBuffer(lmHead.gpuSplitWeight);
|
|
444
|
+
lmHead.gpuSplitWeight = null;
|
|
445
|
+
}
|
|
446
|
+
return null;
|
|
447
|
+
}
|
|
448
|
+
if (lmHead.gpuSplitWeight) {
|
|
449
|
+
return lmHead.gpuSplitWeight;
|
|
450
|
+
}
|
|
451
|
+
if (lmHead.layout !== 'row' || lmHead.dtype !== 'f16') {
|
|
452
|
+
return null;
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
const device = getDevice();
|
|
456
|
+
if (!device) {
|
|
457
|
+
throw new Error('[Logits] GPU device not available for split LM head materialization.');
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
const rowsPerSection = resolveSplitLmHeadRows(device, hiddenSize, largeWeightConfig);
|
|
461
|
+
const createdBuffers = [];
|
|
462
|
+
try {
|
|
463
|
+
const sections = [];
|
|
464
|
+
for (let rowStart = 0; rowStart < weightVocabSize; rowStart += rowsPerSection) {
|
|
465
|
+
const rowCount = Math.min(rowsPerSection, weightVocabSize - rowStart);
|
|
466
|
+
const bytes = await readSplitLmHeadSectionBytes(lmHead, hiddenSize, rowStart, rowCount);
|
|
467
|
+
if (!bytes) {
|
|
468
|
+
for (const buffer of createdBuffers) {
|
|
469
|
+
buffer.destroy();
|
|
470
|
+
}
|
|
471
|
+
return null;
|
|
472
|
+
}
|
|
473
|
+
const buffer = device.createBuffer({
|
|
474
|
+
label: `${lmHead.label ?? 'lm_head'}:lazy_split:${sections.length}`,
|
|
475
|
+
size: alignByteLength(bytes.byteLength),
|
|
476
|
+
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
|
|
477
|
+
});
|
|
478
|
+
createdBuffers.push(buffer);
|
|
479
|
+
writeBufferInChunks(device.queue, buffer, bytes);
|
|
480
|
+
sections.push({ buffer, rowStart, rowCount });
|
|
481
|
+
}
|
|
482
|
+
const splitWeight = createSplitWeightBuffer(
|
|
483
|
+
sections,
|
|
484
|
+
lmHead.dtype,
|
|
485
|
+
lmHead.layout,
|
|
486
|
+
lmHead.shape,
|
|
487
|
+
lmHead.label
|
|
488
|
+
);
|
|
489
|
+
log.warn(
|
|
490
|
+
'Logits',
|
|
491
|
+
`LM head "${lmHead.label ?? 'lm_head'}" materialized as lazy split GPU sections ` +
|
|
492
|
+
`(${sections.length} sections, dtype=${lmHead.dtype}, layout=${lmHead.layout}).`
|
|
493
|
+
);
|
|
494
|
+
Object.defineProperty(lmHead, 'gpuSplitWeight', {
|
|
495
|
+
value: splitWeight,
|
|
496
|
+
configurable: true,
|
|
497
|
+
enumerable: false,
|
|
498
|
+
writable: true,
|
|
499
|
+
});
|
|
500
|
+
return splitWeight;
|
|
501
|
+
} catch (error) {
|
|
502
|
+
for (const buffer of createdBuffers) {
|
|
503
|
+
try {
|
|
504
|
+
buffer.destroy();
|
|
505
|
+
} catch {}
|
|
506
|
+
}
|
|
507
|
+
throw error;
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
|
|
360
512
|
export async function computeChunkedLogitsGPU(
|
|
361
513
|
normedTensor,
|
|
362
514
|
lmHead,
|
|
@@ -378,10 +530,27 @@ export async function computeChunkedLogitsGPU(
|
|
|
378
530
|
throw new Error('[Logits] largeWeights config is required for chunked LM head.');
|
|
379
531
|
}
|
|
380
532
|
|
|
533
|
+
const splitLmHead = await materializeSplitLmHeadGPU(lmHead, hiddenSize, weightVocabSize, largeWeightConfig);
|
|
534
|
+
if (splitLmHead) {
|
|
535
|
+
return computeSplitLogitsGPU(
|
|
536
|
+
normedTensor,
|
|
537
|
+
splitLmHead,
|
|
538
|
+
numTokens,
|
|
539
|
+
hiddenSize,
|
|
540
|
+
vocabSize,
|
|
541
|
+
weightVocabSize,
|
|
542
|
+
debugProbes,
|
|
543
|
+
operatorDiagnostics,
|
|
544
|
+
kernelPath,
|
|
545
|
+
executionPolicies
|
|
546
|
+
);
|
|
547
|
+
}
|
|
548
|
+
|
|
381
549
|
const chunkRows = resolveLmHeadChunkRows(device, numTokens, hiddenSize, largeWeightConfig);
|
|
382
550
|
const phase = numTokens === 1 ? 'decode' : 'prefill';
|
|
383
|
-
const
|
|
384
|
-
const
|
|
551
|
+
const lmHeadRole = resolveLmHeadMatmulRole(phase);
|
|
552
|
+
const lmHeadInputDtype = resolveMatmulStepDtype(lmHeadRole, phase, kernelPath, normedTensor.dtype, 'inputDtype');
|
|
553
|
+
const lmHeadOutputDtype = resolveMatmulStepDtype(lmHeadRole, phase, kernelPath, normedTensor.dtype, 'outputDtype');
|
|
385
554
|
const caps = getKernelCapabilities();
|
|
386
555
|
const weightDtype = selectRuleValue('inference', 'dtype', 'lmHeadChunkWeightDtype', {
|
|
387
556
|
preferF16: largeWeightConfig.preferF16,
|
|
@@ -405,41 +574,52 @@ export async function computeChunkedLogitsGPU(
|
|
|
405
574
|
|
|
406
575
|
for (let rowOffset = 0; rowOffset < vocabSize; rowOffset += chunkRows) {
|
|
407
576
|
const rowCount = Math.min(chunkRows, vocabSize - rowOffset);
|
|
408
|
-
const chunkData = await extractLmHeadChunk(
|
|
409
|
-
lmHead.data,
|
|
410
|
-
lmHead.layout,
|
|
411
|
-
hiddenSize,
|
|
412
|
-
weightVocabSize,
|
|
413
|
-
rowOffset,
|
|
414
|
-
rowCount,
|
|
415
|
-
lmHead.dtype
|
|
416
|
-
);
|
|
417
|
-
|
|
418
|
-
const f32Buffer = acquireBuffer(chunkData.byteLength, undefined, 'lm_head_chunk_f32');
|
|
419
|
-
device.queue.writeBuffer(
|
|
420
|
-
f32Buffer,
|
|
421
|
-
0,
|
|
422
|
-
chunkData.buffer,
|
|
423
|
-
chunkData.byteOffset,
|
|
424
|
-
chunkData.byteLength
|
|
425
|
-
);
|
|
426
|
-
|
|
427
577
|
const chunkShape = lmHead.layout === 'column'
|
|
428
578
|
? [hiddenSize, rowCount]
|
|
429
579
|
: [rowCount, hiddenSize];
|
|
430
580
|
|
|
431
|
-
let weightBuffer
|
|
581
|
+
let weightBuffer;
|
|
582
|
+
if (preferF16 && lmHead.layout === 'row' && lmHead.dtype === 'f16') {
|
|
583
|
+
const chunkBytes = await readSplitLmHeadSectionBytes(lmHead, hiddenSize, rowOffset, rowCount);
|
|
584
|
+
if (!chunkBytes) {
|
|
585
|
+
throw new Error('[Logits] F16 LM head chunk source is not range-readable.');
|
|
586
|
+
}
|
|
587
|
+
const f16Buffer = acquireBuffer(alignByteLength(chunkBytes.byteLength), undefined, 'lm_head_chunk_f16');
|
|
588
|
+
writeBufferInChunks(device.queue, f16Buffer, chunkBytes);
|
|
589
|
+
weightBuffer = createWeightBuffer(f16Buffer, 'f16', lmHead.layout, chunkShape, 'lm_head_chunk_f16');
|
|
590
|
+
} else {
|
|
591
|
+
const chunkData = await extractLmHeadChunk(
|
|
592
|
+
lmHead.data,
|
|
593
|
+
lmHead.layout,
|
|
594
|
+
hiddenSize,
|
|
595
|
+
weightVocabSize,
|
|
596
|
+
rowOffset,
|
|
597
|
+
rowCount,
|
|
598
|
+
lmHead.dtype
|
|
599
|
+
);
|
|
600
|
+
|
|
601
|
+
const f32Buffer = acquireBuffer(chunkData.byteLength, undefined, 'lm_head_chunk_f32');
|
|
602
|
+
device.queue.writeBuffer(
|
|
603
|
+
f32Buffer,
|
|
604
|
+
0,
|
|
605
|
+
chunkData.buffer,
|
|
606
|
+
chunkData.byteOffset,
|
|
607
|
+
chunkData.byteLength
|
|
608
|
+
);
|
|
609
|
+
|
|
610
|
+
weightBuffer = createWeightBuffer(f32Buffer, 'f32', lmHead.layout, chunkShape, 'lm_head_chunk_f32');
|
|
432
611
|
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
612
|
+
if (preferF16) {
|
|
613
|
+
const f32Tensor = createTensor(f32Buffer, 'f32', chunkShape, 'lm_head_chunk_f32');
|
|
614
|
+
const f16Tensor = await castF32ToF16(f32Tensor);
|
|
615
|
+
releaseBuffer(f32Buffer);
|
|
616
|
+
weightBuffer = createWeightBuffer(f16Tensor.buffer, 'f16', lmHead.layout, chunkShape, 'lm_head_chunk_f16');
|
|
617
|
+
}
|
|
438
618
|
}
|
|
439
619
|
|
|
440
620
|
const logitsTensor = await runMatmul(matmulInput, weightBuffer, numTokens, rowCount, hiddenSize, {
|
|
441
621
|
transposeB: 'auto',
|
|
442
|
-
role:
|
|
622
|
+
role: lmHeadRole,
|
|
443
623
|
kernelPath,
|
|
444
624
|
outputDtype: lmHeadOutputDtype,
|
|
445
625
|
executionPolicies,
|
|
@@ -478,6 +658,104 @@ export async function computeChunkedLogitsGPU(
|
|
|
478
658
|
}
|
|
479
659
|
|
|
480
660
|
|
|
661
|
+
export async function computeSplitLogitsGPU(
|
|
662
|
+
normedTensor,
|
|
663
|
+
lmHead,
|
|
664
|
+
numTokens,
|
|
665
|
+
hiddenSize,
|
|
666
|
+
vocabSize,
|
|
667
|
+
weightVocabSize,
|
|
668
|
+
debugProbes,
|
|
669
|
+
operatorDiagnostics,
|
|
670
|
+
kernelPath = null,
|
|
671
|
+
executionPolicies = null
|
|
672
|
+
) {
|
|
673
|
+
const device = getDevice();
|
|
674
|
+
if (!device) {
|
|
675
|
+
throw new Error('[Logits] GPU device not available for split LM head.');
|
|
676
|
+
}
|
|
677
|
+
if (lmHead.layout !== 'row') {
|
|
678
|
+
throw new Error(`[Logits] split LM head requires row layout, got "${lmHead.layout}".`);
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
const phase = numTokens === 1 ? 'decode' : 'prefill';
|
|
682
|
+
const lmHeadRole = resolveLmHeadMatmulRole(phase);
|
|
683
|
+
const lmHeadInputDtype = resolveMatmulStepDtype(lmHeadRole, phase, kernelPath, normedTensor.dtype, 'inputDtype');
|
|
684
|
+
const lmHeadOutputDtype = resolveMatmulStepDtype(lmHeadRole, phase, kernelPath, normedTensor.dtype, 'outputDtype');
|
|
685
|
+
const logits = new Float32Array(numTokens * vocabSize);
|
|
686
|
+
let matmulInput = normedTensor;
|
|
687
|
+
let matmulInputOwned = false;
|
|
688
|
+
|
|
689
|
+
try {
|
|
690
|
+
matmulInput = lmHeadInputDtype !== normedTensor.dtype
|
|
691
|
+
? await coerceTensorDtype(normedTensor, lmHeadInputDtype, null, {
|
|
692
|
+
executionPolicies,
|
|
693
|
+
op: 'lm_head',
|
|
694
|
+
transitionDeclaredBy: 'step_precision',
|
|
695
|
+
})
|
|
696
|
+
: normedTensor;
|
|
697
|
+
matmulInputOwned = matmulInput !== normedTensor;
|
|
698
|
+
|
|
699
|
+
for (const section of lmHead.sections) {
|
|
700
|
+
if (section.rowStart >= vocabSize) {
|
|
701
|
+
continue;
|
|
702
|
+
}
|
|
703
|
+
if (section.rowStart + section.rowCount > weightVocabSize) {
|
|
704
|
+
throw new Error(
|
|
705
|
+
`[Logits] split LM head section exceeds weight vocab: rowStart=${section.rowStart}, ` +
|
|
706
|
+
`rowCount=${section.rowCount}, weightVocabSize=${weightVocabSize}.`
|
|
707
|
+
);
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
const rowCount = Math.min(section.rowCount, vocabSize - section.rowStart);
|
|
711
|
+
const weightBuffer = createWeightBuffer(
|
|
712
|
+
section.buffer,
|
|
713
|
+
lmHead.dtype,
|
|
714
|
+
lmHead.layout,
|
|
715
|
+
[section.rowCount, hiddenSize],
|
|
716
|
+
`${lmHead.label ?? 'lm_head'}:split:${section.rowStart}`
|
|
717
|
+
);
|
|
718
|
+
const logitsTensor = await runMatmul(matmulInput, weightBuffer, numTokens, rowCount, hiddenSize, {
|
|
719
|
+
transposeB: 'auto',
|
|
720
|
+
role: lmHeadRole,
|
|
721
|
+
kernelPath,
|
|
722
|
+
outputDtype: lmHeadOutputDtype,
|
|
723
|
+
executionPolicies,
|
|
724
|
+
});
|
|
725
|
+
|
|
726
|
+
if (debugProbes?.length || operatorDiagnostics?.enabled) {
|
|
727
|
+
await runProbes('logits', logitsTensor.buffer, {
|
|
728
|
+
numTokens,
|
|
729
|
+
hiddenSize: rowCount,
|
|
730
|
+
probes: debugProbes,
|
|
731
|
+
operatorDiagnostics,
|
|
732
|
+
dtype: logitsTensor.dtype,
|
|
733
|
+
});
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: logitsTensor.dtype });
|
|
737
|
+
const chunkLogitsData = await readBufferWithCleanup(
|
|
738
|
+
logitsTensor.buffer,
|
|
739
|
+
numTokens * rowCount * logitsBytes,
|
|
740
|
+
() => {
|
|
741
|
+
releaseBuffer(logitsTensor.buffer);
|
|
742
|
+
}
|
|
743
|
+
);
|
|
744
|
+
const chunkLogits = logitsTensor.dtype === 'f16'
|
|
745
|
+
? f16BufferToF32(chunkLogitsData)
|
|
746
|
+
: new Float32Array(chunkLogitsData);
|
|
747
|
+
writeChunkLogits(logits, chunkLogits, numTokens, vocabSize, section.rowStart, rowCount);
|
|
748
|
+
}
|
|
749
|
+
} finally {
|
|
750
|
+
if (matmulInputOwned) {
|
|
751
|
+
releaseBuffer(matmulInput.buffer);
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
return logits;
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
|
|
481
759
|
export async function computeLogitsGPU(
|
|
482
760
|
hiddenStates,
|
|
483
761
|
numTokens,
|
|
@@ -508,7 +786,7 @@ export async function computeLogitsGPU(
|
|
|
508
786
|
log.warn('Pipeline', 'Final norm or LM head not loaded');
|
|
509
787
|
return null;
|
|
510
788
|
}
|
|
511
|
-
if (isCpuWeightBuffer(lmHead)) {
|
|
789
|
+
if (isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead)) {
|
|
512
790
|
return null;
|
|
513
791
|
}
|
|
514
792
|
|
|
@@ -621,12 +899,13 @@ export async function computeLogitsGPU(
|
|
|
621
899
|
releaseBuffer(normInputTensor.buffer);
|
|
622
900
|
normInputOwned = false;
|
|
623
901
|
}
|
|
902
|
+
const lmHeadRole = resolveLmHeadMatmulRole(phase);
|
|
624
903
|
const lmHeadInputDtype = forceStableF32Logits
|
|
625
904
|
? finalNormTensor.dtype
|
|
626
|
-
: resolveMatmulStepDtype(
|
|
905
|
+
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, finalNormTensor.dtype, 'inputDtype');
|
|
627
906
|
const lmHeadOutputDtype = forceStableF32Logits
|
|
628
907
|
? finalNormTensor.dtype
|
|
629
|
-
: resolveMatmulStepDtype(
|
|
908
|
+
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, finalNormTensor.dtype, 'outputDtype');
|
|
630
909
|
lmHeadInputTensor = lmHeadInputDtype !== finalNormTensor.dtype
|
|
631
910
|
? await coerceTensorDtype(finalNormTensor, lmHeadInputDtype, null, {
|
|
632
911
|
executionPolicies: config.executionPolicies ?? null,
|
|
@@ -654,7 +933,7 @@ export async function computeLogitsGPU(
|
|
|
654
933
|
|
|
655
934
|
const logitsTensor = await runMatmul(lmHeadInputTensor, lmHeadBuffer, numTokens, matmulVocabSize, hiddenSize, {
|
|
656
935
|
transposeB: 'auto',
|
|
657
|
-
role:
|
|
936
|
+
role: lmHeadRole,
|
|
658
937
|
kernelPath: stableKernelPath,
|
|
659
938
|
outputDtype: lmHeadOutputDtype,
|
|
660
939
|
executionPolicies: config.executionPolicies ?? null,
|
|
@@ -690,7 +969,7 @@ export async function computeLogitsGPU(
|
|
|
690
969
|
}
|
|
691
970
|
|
|
692
971
|
|
|
693
|
-
|
|
972
|
+
async function recordLogitsTailGPU(
|
|
694
973
|
recorder,
|
|
695
974
|
hiddenStates,
|
|
696
975
|
numTokens,
|
|
@@ -712,8 +991,8 @@ export async function recordLogitsGPU(
|
|
|
712
991
|
if (!finalNorm || !lmHead) {
|
|
713
992
|
throw new Error('[recordLogitsGPU] Final norm or LM head not loaded');
|
|
714
993
|
}
|
|
715
|
-
if (isCpuWeightBuffer(lmHead)) {
|
|
716
|
-
throw new Error('[recordLogitsGPU] CPU-resident LM head not supported in recorded path');
|
|
994
|
+
if (isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead)) {
|
|
995
|
+
throw new Error('[recordLogitsGPU] CPU-resident or split LM head not supported in recorded path');
|
|
717
996
|
}
|
|
718
997
|
|
|
719
998
|
// Get norm weight buffer
|
|
@@ -775,6 +1054,7 @@ export async function recordLogitsGPU(
|
|
|
775
1054
|
batchSize: numTokens,
|
|
776
1055
|
hiddenSize,
|
|
777
1056
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
1057
|
+
label: 'final_norm',
|
|
778
1058
|
});
|
|
779
1059
|
let finalNormTensor = normedTensor;
|
|
780
1060
|
if (!forceStableF32Logits) {
|
|
@@ -800,12 +1080,13 @@ export async function recordLogitsGPU(
|
|
|
800
1080
|
operatorDiagnostics,
|
|
801
1081
|
dtype: finalNormTensor.dtype,
|
|
802
1082
|
});
|
|
1083
|
+
const lmHeadRole = resolveLmHeadMatmulRole(phase);
|
|
803
1084
|
const lmHeadInputDtype = forceStableF32Logits
|
|
804
1085
|
? finalNormTensor.dtype
|
|
805
|
-
: resolveMatmulStepDtype(
|
|
1086
|
+
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, finalNormTensor.dtype, 'inputDtype');
|
|
806
1087
|
const lmHeadOutputDtype = forceStableF32Logits
|
|
807
1088
|
? finalNormTensor.dtype
|
|
808
|
-
: resolveMatmulStepDtype(
|
|
1089
|
+
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, finalNormTensor.dtype, 'outputDtype');
|
|
809
1090
|
const lmHeadInputTensor = lmHeadInputDtype !== finalNormTensor.dtype
|
|
810
1091
|
? await coerceTensorDtype(finalNormTensor, lmHeadInputDtype, recorder, {
|
|
811
1092
|
executionPolicies: config.executionPolicies ?? null,
|
|
@@ -829,23 +1110,26 @@ export async function recordLogitsGPU(
|
|
|
829
1110
|
lmHeadBufferOwned = true;
|
|
830
1111
|
}
|
|
831
1112
|
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
1113
|
+
return {
|
|
1114
|
+
hiddenSize,
|
|
1115
|
+
matmulVocabSize,
|
|
1116
|
+
phase,
|
|
1117
|
+
stableKernelPath,
|
|
1118
|
+
lmHeadRole,
|
|
1119
|
+
lmHeadOutputDtype,
|
|
1120
|
+
normedTensor,
|
|
1121
|
+
finalNormTensor,
|
|
1122
|
+
normInputTensor,
|
|
1123
|
+
normInputOwned,
|
|
1124
|
+
normWeightBuffer,
|
|
1125
|
+
normWeightOwned,
|
|
1126
|
+
lmHeadInputTensor,
|
|
1127
|
+
lmHeadBuffer,
|
|
1128
|
+
lmHeadBufferOwned,
|
|
1129
|
+
};
|
|
1130
|
+
}
|
|
847
1131
|
|
|
848
|
-
|
|
1132
|
+
function trackRecordedLogitsTail(recorder, tail) {
|
|
849
1133
|
const trackedTempBuffers = new Set();
|
|
850
1134
|
const trackTempBufferOnce = (buffer) => {
|
|
851
1135
|
if (!buffer || trackedTempBuffers.has(buffer)) {
|
|
@@ -854,22 +1138,89 @@ export async function recordLogitsGPU(
|
|
|
854
1138
|
trackedTempBuffers.add(buffer);
|
|
855
1139
|
recorder.trackTemporaryBuffer(buffer);
|
|
856
1140
|
};
|
|
857
|
-
if (finalNormTensor !== normedTensor) {
|
|
858
|
-
trackTempBufferOnce(normedTensor.buffer);
|
|
1141
|
+
if (tail.finalNormTensor !== tail.normedTensor) {
|
|
1142
|
+
trackTempBufferOnce(tail.normedTensor.buffer);
|
|
859
1143
|
}
|
|
860
|
-
trackTempBufferOnce(finalNormTensor.buffer);
|
|
861
|
-
if (lmHeadInputTensor !== finalNormTensor) {
|
|
862
|
-
trackTempBufferOnce(lmHeadInputTensor.buffer);
|
|
1144
|
+
trackTempBufferOnce(tail.finalNormTensor.buffer);
|
|
1145
|
+
if (tail.lmHeadInputTensor !== tail.finalNormTensor) {
|
|
1146
|
+
trackTempBufferOnce(tail.lmHeadInputTensor.buffer);
|
|
863
1147
|
}
|
|
864
|
-
if (normWeightOwned) {
|
|
865
|
-
recorder.trackTemporaryBuffer(normWeightBuffer);
|
|
1148
|
+
if (tail.normWeightOwned) {
|
|
1149
|
+
recorder.trackTemporaryBuffer(tail.normWeightBuffer);
|
|
866
1150
|
}
|
|
867
|
-
if (normInputOwned) {
|
|
868
|
-
recorder.trackTemporaryBuffer(normInputTensor.buffer);
|
|
1151
|
+
if (tail.normInputOwned) {
|
|
1152
|
+
recorder.trackTemporaryBuffer(tail.normInputTensor.buffer);
|
|
869
1153
|
}
|
|
870
|
-
if (lmHeadBufferOwned) {
|
|
871
|
-
recorder.trackTemporaryBuffer(isWeightBuffer(lmHeadBuffer) ? lmHeadBuffer.buffer : lmHeadBuffer);
|
|
1154
|
+
if (tail.lmHeadBufferOwned) {
|
|
1155
|
+
recorder.trackTemporaryBuffer(isWeightBuffer(tail.lmHeadBuffer) ? tail.lmHeadBuffer.buffer : tail.lmHeadBuffer);
|
|
872
1156
|
}
|
|
1157
|
+
}
|
|
873
1158
|
|
|
874
|
-
|
|
1159
|
+
export async function recordLogitsGPU(
|
|
1160
|
+
recorder,
|
|
1161
|
+
hiddenStates,
|
|
1162
|
+
numTokens,
|
|
1163
|
+
weights,
|
|
1164
|
+
config,
|
|
1165
|
+
operatorDiagnostics = null,
|
|
1166
|
+
) {
|
|
1167
|
+
const tail = await recordLogitsTailGPU(
|
|
1168
|
+
recorder,
|
|
1169
|
+
hiddenStates,
|
|
1170
|
+
numTokens,
|
|
1171
|
+
weights,
|
|
1172
|
+
config,
|
|
1173
|
+
operatorDiagnostics
|
|
1174
|
+
);
|
|
1175
|
+
|
|
1176
|
+
const logitsTensor = await recordMatmul(recorder, tail.lmHeadInputTensor, tail.lmHeadBuffer, numTokens, tail.matmulVocabSize, tail.hiddenSize, {
|
|
1177
|
+
transposeB: 'auto',
|
|
1178
|
+
role: tail.lmHeadRole,
|
|
1179
|
+
kernelPath: tail.stableKernelPath,
|
|
1180
|
+
outputDtype: tail.lmHeadOutputDtype,
|
|
1181
|
+
executionPolicies: config.executionPolicies ?? null,
|
|
1182
|
+
});
|
|
1183
|
+
await runProbes('logits', logitsTensor.buffer, {
|
|
1184
|
+
numTokens,
|
|
1185
|
+
hiddenSize: tail.matmulVocabSize,
|
|
1186
|
+
recorder,
|
|
1187
|
+
operatorDiagnostics,
|
|
1188
|
+
dtype: logitsTensor.dtype,
|
|
1189
|
+
});
|
|
1190
|
+
|
|
1191
|
+
trackRecordedLogitsTail(recorder, tail);
|
|
1192
|
+
|
|
1193
|
+
return { logitsBuffer: logitsTensor.buffer, vocabSize: tail.matmulVocabSize, logitsDtype: logitsTensor.dtype };
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
export async function recordGreedyLmHeadArgmaxGPU(
|
|
1197
|
+
recorder,
|
|
1198
|
+
hiddenStates,
|
|
1199
|
+
numTokens,
|
|
1200
|
+
weights,
|
|
1201
|
+
config,
|
|
1202
|
+
options,
|
|
1203
|
+
operatorDiagnostics = null,
|
|
1204
|
+
) {
|
|
1205
|
+
if (numTokens !== 1) {
|
|
1206
|
+
throw new Error(`[recordGreedyLmHeadArgmaxGPU] expected numTokens=1, got ${numTokens}.`);
|
|
1207
|
+
}
|
|
1208
|
+
const tail = await recordLogitsTailGPU(
|
|
1209
|
+
recorder,
|
|
1210
|
+
hiddenStates,
|
|
1211
|
+
numTokens,
|
|
1212
|
+
weights,
|
|
1213
|
+
config,
|
|
1214
|
+
operatorDiagnostics
|
|
1215
|
+
);
|
|
1216
|
+
const outputBuffer = await recordLmHeadArgmaxF16(recorder, tail.lmHeadInputTensor, tail.lmHeadBuffer, {
|
|
1217
|
+
vocabSize: tail.matmulVocabSize,
|
|
1218
|
+
hiddenSize: tail.hiddenSize,
|
|
1219
|
+
padTokenId: options.padTokenId,
|
|
1220
|
+
logitSoftcap: options.logitSoftcap,
|
|
1221
|
+
outputBuffer: options.outputBuffer,
|
|
1222
|
+
outputIndex: options.outputIndex,
|
|
1223
|
+
});
|
|
1224
|
+
trackRecordedLogitsTail(recorder, tail);
|
|
1225
|
+
return outputBuffer;
|
|
875
1226
|
}
|
|
@@ -20,7 +20,7 @@ export type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
|
|
|
20
20
|
export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
|
|
21
21
|
|
|
22
22
|
// Re-export GPU functions
|
|
23
|
-
export { computeLogitsGPU, recordLogitsGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
|
|
23
|
+
export { computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
|
|
24
24
|
|
|
25
25
|
// Re-export utilities
|
|
26
26
|
export { extractLastPositionLogits, finalizeLogits, readBufferWithCleanup } from './utils.js';
|
|
@@ -32,7 +32,7 @@ export interface ComputeLogitsOptions {
|
|
|
32
32
|
export interface ResolvedLmHeadMatmulConfig {
|
|
33
33
|
lastPositionOnly: boolean;
|
|
34
34
|
matmulRows: number;
|
|
35
|
-
phaseOverride: 'decode' | null;
|
|
35
|
+
phaseOverride: 'decode' | 'prefill' | null;
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
export function resolveLmHeadMatmulConfig(
|