doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -1,24 +1,31 @@
|
|
|
1
1
|
import { getDevice, getKernelCapabilities } from '../../../gpu/device.js';
|
|
2
|
-
import { acquireBuffer, releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
|
|
2
|
+
import { acquireBuffer, BufferUsage, releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
|
|
3
3
|
import { createTensor } from '../../../gpu/tensor.js';
|
|
4
4
|
import { castF16ToF32, castF32ToF16 } from '../../../gpu/kernels/cast.js';
|
|
5
5
|
import {
|
|
6
6
|
runMatmul,
|
|
7
7
|
runSiLU,
|
|
8
|
+
runSiLURowSplit,
|
|
8
9
|
runGeLU,
|
|
9
10
|
dequantizeMXFP4Expert,
|
|
10
11
|
runBiasAdd,
|
|
12
|
+
runRMSNorm,
|
|
13
|
+
runScale,
|
|
11
14
|
runSoftmaxTopK,
|
|
12
15
|
runMoEGather,
|
|
13
16
|
runMoEBuildTokenOffsets,
|
|
14
17
|
runScatterAddDynamic,
|
|
15
18
|
runSwiGLURowsplitBias,
|
|
19
|
+
runGemma4RouteQ4MatmulF16A,
|
|
20
|
+
runScatterAddRoutesF16ExpertScale,
|
|
16
21
|
} from '../../../gpu/kernel-selector.js';
|
|
22
|
+
import { getBuffer, getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
17
23
|
import { trace, isTraceEnabled } from '../../../debug/index.js';
|
|
18
24
|
import { f16ToF32Array } from '../../kv-cache/types.js';
|
|
19
25
|
import { resolveMaxTokensPerExpert, getCachedDequant, setCachedDequant, getDequantCacheStats } from './moe-cache.js';
|
|
20
26
|
import { ensureExpertLoaded } from './moe-helpers.js';
|
|
21
27
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
28
|
+
import { getRuntimeConfig } from '../../../config/runtime.js';
|
|
22
29
|
import {
|
|
23
30
|
validateMoeShape,
|
|
24
31
|
resolveMoeVendorProfile,
|
|
@@ -26,6 +33,100 @@ import {
|
|
|
26
33
|
resolveMixtralKernelPathProfile,
|
|
27
34
|
} from './moe-shape-validator.js';
|
|
28
35
|
import { assertImplicitDtypeTransitionAllowed } from './dtype-contract.js';
|
|
36
|
+
import { QK_K, Q4K_BLOCK_BYTES } from '../../../config/schema/index.js';
|
|
37
|
+
|
|
38
|
+
function resolveMoEActiveExpertSelection() {
|
|
39
|
+
const selection = getRuntimeConfig()?.inference?.moe?.routing?.activeExpertSelection;
|
|
40
|
+
if (selection === 'all' || selection === 'topk-readback' || selection === 'topk-route') {
|
|
41
|
+
return selection;
|
|
42
|
+
}
|
|
43
|
+
throw new Error(
|
|
44
|
+
'[MoE] runtime.inference.moe.routing.activeExpertSelection must be ' +
|
|
45
|
+
`"all", "topk-readback", or "topk-route", got ${String(selection)}.`
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async function resolveActiveExpertSchedule(indicesBuffer, numTokens, numExperts, topK, maxTokensPerExpert) {
|
|
50
|
+
const selection = resolveMoEActiveExpertSelection();
|
|
51
|
+
if (selection === 'all') {
|
|
52
|
+
return {
|
|
53
|
+
selection,
|
|
54
|
+
activeExperts: Array.from({ length: numExperts }, (_, expertIdx) => expertIdx),
|
|
55
|
+
tokenCounts: null,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const indicesBytes = numTokens * topK * 4;
|
|
60
|
+
const indicesData = await readBuffer(indicesBuffer, indicesBytes);
|
|
61
|
+
return buildActiveExpertScheduleFromIndices(
|
|
62
|
+
new Uint32Array(indicesData),
|
|
63
|
+
numExperts,
|
|
64
|
+
maxTokensPerExpert,
|
|
65
|
+
selection
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function buildActiveExpertScheduleFromIndices(
|
|
70
|
+
indices,
|
|
71
|
+
numExperts,
|
|
72
|
+
maxTokensPerExpert,
|
|
73
|
+
selection = 'topk-readback'
|
|
74
|
+
) {
|
|
75
|
+
const tokenCounts = new Uint32Array(numExperts);
|
|
76
|
+
|
|
77
|
+
for (let i = 0; i < indices.length; i++) {
|
|
78
|
+
const expertIdx = indices[i];
|
|
79
|
+
if (expertIdx >= numExperts) {
|
|
80
|
+
throw new Error(
|
|
81
|
+
`[MoE] Top-K routing produced expert index ${expertIdx} outside numExperts=${numExperts}.`
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
tokenCounts[expertIdx] += 1;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const activeExperts = [];
|
|
88
|
+
for (let expertIdx = 0; expertIdx < numExperts; expertIdx++) {
|
|
89
|
+
const count = tokenCounts[expertIdx];
|
|
90
|
+
if (count === 0) {
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
if (count > maxTokensPerExpert) {
|
|
94
|
+
throw new Error(
|
|
95
|
+
`[MoE] Expert ${expertIdx} received ${count} tokens but maxTokensPerExpert=${maxTokensPerExpert}. ` +
|
|
96
|
+
'Increase runtime.inference.moe.routing.maxTokensPerExpert or its headroom/cap settings.'
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
activeExperts.push(expertIdx);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
return { selection, activeExperts, tokenCounts };
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function resolvePerExpertScaleBuffer(device, value) {
|
|
106
|
+
if (value == null) {
|
|
107
|
+
return { buffer: null, ownedBuffer: null };
|
|
108
|
+
}
|
|
109
|
+
if (value instanceof Float32Array) {
|
|
110
|
+
const buffer = acquireBuffer(value.byteLength, BufferUsage.STORAGE_READ, 'moe_per_expert_scale_f32');
|
|
111
|
+
try {
|
|
112
|
+
device.queue.writeBuffer(buffer, 0, value);
|
|
113
|
+
} catch (error) {
|
|
114
|
+
releaseBuffer(buffer);
|
|
115
|
+
throw error;
|
|
116
|
+
}
|
|
117
|
+
return { buffer, ownedBuffer: buffer };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const dtype = getWeightDtype(value);
|
|
121
|
+
if (dtype != null && dtype !== 'f32') {
|
|
122
|
+
throw new Error(`[MoE] per-expert router scale must be f32 for scatter-add, got ${dtype}.`);
|
|
123
|
+
}
|
|
124
|
+
const buffer = getBuffer(value);
|
|
125
|
+
if (!isGpuBufferInstance(buffer)) {
|
|
126
|
+
throw new Error('[MoE] per-expert router scale must resolve to a GPUBuffer.');
|
|
127
|
+
}
|
|
128
|
+
return { buffer, ownedBuffer: null };
|
|
129
|
+
}
|
|
29
130
|
|
|
30
131
|
export async function moeFeedForwardGPU(
|
|
31
132
|
inputBuffer,
|
|
@@ -40,8 +141,11 @@ export async function moeFeedForwardGPU(
|
|
|
40
141
|
const device = getDevice();
|
|
41
142
|
if (!device) throw new Error('No GPU device for MoE');
|
|
42
143
|
|
|
43
|
-
const { hiddenSize, numExperts,
|
|
144
|
+
const { hiddenSize, numExperts, moeTopK, hiddenActivation } = config;
|
|
44
145
|
const expertFormat = config.expertFormat;
|
|
146
|
+
const intermediateSize = expertFormat === 'gemma4'
|
|
147
|
+
? config.expertIntermediateSize
|
|
148
|
+
: config.intermediateSize;
|
|
45
149
|
const swigluLimit = config.swigluLimit;
|
|
46
150
|
const kernelPath = config.kernelPath ?? null;
|
|
47
151
|
if (!expertFormat) {
|
|
@@ -50,6 +154,9 @@ export async function moeFeedForwardGPU(
|
|
|
50
154
|
if (swigluLimit === undefined) {
|
|
51
155
|
throw new Error('MoE swigluLimit must be explicitly set (null or number).');
|
|
52
156
|
}
|
|
157
|
+
if (!Number.isFinite(intermediateSize) || intermediateSize <= 0) {
|
|
158
|
+
throw new Error(`[MoE] Invalid expert intermediate size for ${expertFormat}: ${String(intermediateSize)}.`);
|
|
159
|
+
}
|
|
53
160
|
const topK = moeTopK ?? moeRouter.topK;
|
|
54
161
|
if (topK == null) {
|
|
55
162
|
throw new Error('MoE topK is required in config.');
|
|
@@ -86,6 +193,12 @@ export async function moeFeedForwardGPU(
|
|
|
86
193
|
};
|
|
87
194
|
|
|
88
195
|
const inputTensor = createTensor(inputBuffer, activationDtype, [numTokens, hiddenSize], 'moe_input');
|
|
196
|
+
const routerSourceTensor = createTensor(
|
|
197
|
+
config.routerInputBuffer ?? inputBuffer,
|
|
198
|
+
config.routerInputDtype ?? activationDtype,
|
|
199
|
+
[numTokens, hiddenSize],
|
|
200
|
+
'moe_router_input'
|
|
201
|
+
);
|
|
89
202
|
let logitsBuffer = null;
|
|
90
203
|
let indicesBuffer = null;
|
|
91
204
|
let weightsBuffer = null;
|
|
@@ -95,18 +208,65 @@ export async function moeFeedForwardGPU(
|
|
|
95
208
|
let expertOutputs = null;
|
|
96
209
|
let tokenOffsets = null;
|
|
97
210
|
let outputTensor = null;
|
|
211
|
+
let routerNormTensor = null;
|
|
212
|
+
let routerScaledTensor = null;
|
|
213
|
+
let ownedPerExpertScaleBuffer = null;
|
|
214
|
+
let activeExpertSchedule = null;
|
|
98
215
|
|
|
99
216
|
const layerRouter = layerRouterWeights?.get(layerIdx) || null;
|
|
100
217
|
if (layerRouter) {
|
|
101
|
-
moeRouter.loadWeights(
|
|
218
|
+
moeRouter.loadWeights(
|
|
219
|
+
layerRouter.weight,
|
|
220
|
+
layerRouter.bias || null,
|
|
221
|
+
layerRouter.scale || null,
|
|
222
|
+
layerRouter.perExpertScale || null
|
|
223
|
+
);
|
|
102
224
|
}
|
|
103
225
|
|
|
104
226
|
try {
|
|
227
|
+
const needsGemmaRouterScale = modelType === 'diffusion_gemma'
|
|
228
|
+
|| layerRouter?.scale != null
|
|
229
|
+
|| layerRouter?.perExpertScale != null;
|
|
230
|
+
let routerInputTensor = routerSourceTensor;
|
|
231
|
+
if (needsGemmaRouterScale) {
|
|
232
|
+
if (!layerRouter?.scale) {
|
|
233
|
+
throw new Error(`[MoE] DiffusionGemma router scale missing for layer ${layerIdx}.`);
|
|
234
|
+
}
|
|
235
|
+
if (!layerRouter?.perExpertScale) {
|
|
236
|
+
throw new Error(`[MoE] DiffusionGemma per-expert router scale missing for layer ${layerIdx}.`);
|
|
237
|
+
}
|
|
238
|
+
if (!Number.isFinite(config.rmsNormEps) || config.rmsNormEps <= 0) {
|
|
239
|
+
throw new Error(`[MoE] DiffusionGemma router RMSNorm eps is invalid: ${String(config.rmsNormEps)}.`);
|
|
240
|
+
}
|
|
241
|
+
routerNormTensor = await runRMSNorm(
|
|
242
|
+
inputTensor,
|
|
243
|
+
layerRouter.scale,
|
|
244
|
+
config.rmsNormEps,
|
|
245
|
+
{
|
|
246
|
+
batchSize: numTokens,
|
|
247
|
+
hiddenSize,
|
|
248
|
+
rmsNormWeightOffset: false,
|
|
249
|
+
}
|
|
250
|
+
);
|
|
251
|
+
routerScaledTensor = await runScale(
|
|
252
|
+
routerNormTensor,
|
|
253
|
+
1 / Math.sqrt(hiddenSize),
|
|
254
|
+
{ count: numTokens * hiddenSize }
|
|
255
|
+
);
|
|
256
|
+
releaseBuffer(routerNormTensor.buffer);
|
|
257
|
+
routerNormTensor = null;
|
|
258
|
+
routerInputTensor = routerScaledTensor;
|
|
259
|
+
}
|
|
260
|
+
|
|
105
261
|
let stepStart = perfMark();
|
|
106
|
-
logitsBuffer = await moeRouter.computeRouterLogitsGPU(
|
|
107
|
-
inputDtype:
|
|
262
|
+
logitsBuffer = await moeRouter.computeRouterLogitsGPU(routerInputTensor.buffer, numTokens, null, {
|
|
263
|
+
inputDtype: routerInputTensor.dtype,
|
|
108
264
|
outputDtype: activationDtype,
|
|
109
265
|
});
|
|
266
|
+
if (routerScaledTensor) {
|
|
267
|
+
releaseBuffer(routerScaledTensor.buffer);
|
|
268
|
+
routerScaledTensor = null;
|
|
269
|
+
}
|
|
110
270
|
const logitsDtype = moeRouter.lastLogitsDtype ?? activationDtype;
|
|
111
271
|
perfLog(`MoE L${layerIdx} router`, stepStart, { numTokens, logitsDtype });
|
|
112
272
|
|
|
@@ -227,6 +387,40 @@ export async function moeFeedForwardGPU(
|
|
|
227
387
|
releaseBuffer(logitsBuffer);
|
|
228
388
|
logitsBuffer = null;
|
|
229
389
|
|
|
390
|
+
const activeExpertSelection = resolveMoEActiveExpertSelection();
|
|
391
|
+
if (activeExpertSelection === 'topk-route') {
|
|
392
|
+
if (expertFormat !== 'gemma4') {
|
|
393
|
+
throw new Error(`[MoE] topk-route active expert selection requires gemma4 expert format, got ${expertFormat}.`);
|
|
394
|
+
}
|
|
395
|
+
stepStart = perfMark();
|
|
396
|
+
await ensureExpertLoaded(layerIdx, 0, expertWeights, expertLoader);
|
|
397
|
+
const routeWeights = expertWeights.get(`layer_${layerIdx}_expert_0`);
|
|
398
|
+
perfLog(`MoE L${layerIdx} route_weight_load`, stepStart, { expertFormat, topK });
|
|
399
|
+
stepStart = perfMark();
|
|
400
|
+
outputTensor = await runGemma4RouteExperts(
|
|
401
|
+
inputTensor,
|
|
402
|
+
indicesBuffer,
|
|
403
|
+
weightsBuffer,
|
|
404
|
+
layerRouter,
|
|
405
|
+
routeWeights,
|
|
406
|
+
layerIdx,
|
|
407
|
+
numTokens,
|
|
408
|
+
topK,
|
|
409
|
+
hiddenSize,
|
|
410
|
+
intermediateSize,
|
|
411
|
+
activationDtype,
|
|
412
|
+
swigluLimit
|
|
413
|
+
);
|
|
414
|
+
perfLog(`MoE L${layerIdx} route_experts`, stepStart, {
|
|
415
|
+
numTokens,
|
|
416
|
+
topK,
|
|
417
|
+
numRoutes: numTokens * topK,
|
|
418
|
+
hiddenSize,
|
|
419
|
+
intermediateSize,
|
|
420
|
+
});
|
|
421
|
+
return outputTensor.buffer;
|
|
422
|
+
}
|
|
423
|
+
|
|
230
424
|
const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
|
|
231
425
|
const bytesPerToken = hiddenSize * bytesPerElement;
|
|
232
426
|
let maxTokensPerExpert = resolveMaxTokensPerExpert(numTokens, numExperts, topK, hiddenSize, activationDtype);
|
|
@@ -237,6 +431,21 @@ export async function moeFeedForwardGPU(
|
|
|
237
431
|
);
|
|
238
432
|
}
|
|
239
433
|
|
|
434
|
+
stepStart = perfMark();
|
|
435
|
+
activeExpertSchedule = await resolveActiveExpertSchedule(
|
|
436
|
+
indicesBuffer,
|
|
437
|
+
numTokens,
|
|
438
|
+
numExperts,
|
|
439
|
+
topK,
|
|
440
|
+
maxTokensPerExpert
|
|
441
|
+
);
|
|
442
|
+
perfLog(`MoE L${layerIdx} active_experts`, stepStart, {
|
|
443
|
+
selection: activeExpertSchedule.selection,
|
|
444
|
+
activeExperts: activeExpertSchedule.activeExperts.length,
|
|
445
|
+
numExperts,
|
|
446
|
+
maxTokensPerExpert,
|
|
447
|
+
});
|
|
448
|
+
|
|
240
449
|
stepStart = perfMark();
|
|
241
450
|
({ gathered, tokenCounts, tokenMap } = await runMoEGather(
|
|
242
451
|
inputTensor,
|
|
@@ -255,10 +464,6 @@ export async function moeFeedForwardGPU(
|
|
|
255
464
|
'moe_expert_outputs_gathered'
|
|
256
465
|
);
|
|
257
466
|
|
|
258
|
-
const zeroEncoder = device.createCommandEncoder({ label: 'zero_moe_expert_outputs' });
|
|
259
|
-
zeroEncoder.clearBuffer(expertOutputs, 0, numExperts * maxTokensPerExpert * hiddenSize * bytesPerElement);
|
|
260
|
-
device.queue.submit([zeroEncoder.finish()]);
|
|
261
|
-
|
|
262
467
|
stepStart = perfMark();
|
|
263
468
|
tokenOffsets = await runMoEBuildTokenOffsets(
|
|
264
469
|
tokenCounts,
|
|
@@ -279,11 +484,13 @@ export async function moeFeedForwardGPU(
|
|
|
279
484
|
const expertStrideBytes = maxTokensPerExpert * bytesPerToken;
|
|
280
485
|
const rowsPerExpert = maxTokensPerExpert;
|
|
281
486
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
487
|
+
const scheduledExperts = activeExpertSchedule?.activeExperts ?? [];
|
|
488
|
+
const scheduledTokenCounts = activeExpertSchedule?.tokenCounts ?? null;
|
|
489
|
+
for (const expertIdx of scheduledExperts) {
|
|
490
|
+
const count = scheduledTokenCounts ? scheduledTokenCounts[expertIdx] : rowsPerExpert;
|
|
491
|
+
if (count <= 0) {
|
|
492
|
+
continue;
|
|
493
|
+
}
|
|
287
494
|
|
|
288
495
|
stepStart = perfMark();
|
|
289
496
|
await ensureExpertLoaded(layerIdx, expertIdx, expertWeights, expertLoader);
|
|
@@ -328,6 +535,22 @@ export async function moeFeedForwardGPU(
|
|
|
328
535
|
vendorProfile,
|
|
329
536
|
gptOssKernelPathProfile
|
|
330
537
|
);
|
|
538
|
+
} else if (expertFormat === 'gemma4' && weights.gateUp && weights.down) {
|
|
539
|
+
await runGemma4Expert(
|
|
540
|
+
gathered,
|
|
541
|
+
expertOutputs,
|
|
542
|
+
weights,
|
|
543
|
+
count,
|
|
544
|
+
inputOffset,
|
|
545
|
+
outputOffset,
|
|
546
|
+
hiddenSize,
|
|
547
|
+
intermediateSize,
|
|
548
|
+
activationDtype,
|
|
549
|
+
swigluLimit,
|
|
550
|
+
kernelPath
|
|
551
|
+
);
|
|
552
|
+
} else if (expertFormat === 'gemma4') {
|
|
553
|
+
throw new Error(`[MoE] Missing Gemma-style packed weights for ${expertKey}`);
|
|
331
554
|
} else if (expertFormat === 'mixtral' && weights.gate && weights.up && weights.down) {
|
|
332
555
|
await runMixtralExpert(
|
|
333
556
|
gathered,
|
|
@@ -355,6 +578,8 @@ export async function moeFeedForwardGPU(
|
|
|
355
578
|
[numExperts, maxTokensPerExpert, hiddenSize],
|
|
356
579
|
'moe_expert_outputs'
|
|
357
580
|
);
|
|
581
|
+
const perExpertScale = resolvePerExpertScaleBuffer(device, layerRouter?.perExpertScale || null);
|
|
582
|
+
ownedPerExpertScaleBuffer = perExpertScale.ownedBuffer;
|
|
358
583
|
stepStart = perfMark();
|
|
359
584
|
outputTensor = await runScatterAddDynamic(
|
|
360
585
|
expertOutputsTensor,
|
|
@@ -364,7 +589,10 @@ export async function moeFeedForwardGPU(
|
|
|
364
589
|
numTokens,
|
|
365
590
|
hiddenSize,
|
|
366
591
|
topK,
|
|
367
|
-
{
|
|
592
|
+
{
|
|
593
|
+
weightsDtype: activationDtype,
|
|
594
|
+
perExpertScale: perExpertScale.buffer,
|
|
595
|
+
}
|
|
368
596
|
);
|
|
369
597
|
perfLog(`MoE L${layerIdx} scatter`, stepStart, { numTokens, hiddenSize });
|
|
370
598
|
|
|
@@ -386,7 +614,9 @@ export async function moeFeedForwardGPU(
|
|
|
386
614
|
trace.perf(`MoE L${layerIdx} done`, {
|
|
387
615
|
numTokens,
|
|
388
616
|
topK,
|
|
389
|
-
executedExperts:
|
|
617
|
+
executedExperts: scheduledExperts.length,
|
|
618
|
+
activeExperts: scheduledExperts.length,
|
|
619
|
+
activeExpertSelection: activeExpertSchedule?.selection ?? null,
|
|
390
620
|
rowsPerExpert,
|
|
391
621
|
maxTokensPerExpert,
|
|
392
622
|
dequantCacheHits: cacheStats.hits,
|
|
@@ -400,6 +630,8 @@ export async function moeFeedForwardGPU(
|
|
|
400
630
|
return outputTensor.buffer;
|
|
401
631
|
} finally {
|
|
402
632
|
if (logitsBuffer) releaseBuffer(logitsBuffer);
|
|
633
|
+
if (routerNormTensor?.buffer) releaseBuffer(routerNormTensor.buffer);
|
|
634
|
+
if (routerScaledTensor?.buffer) releaseBuffer(routerScaledTensor.buffer);
|
|
403
635
|
if (tokenCounts) releaseBuffer(tokenCounts);
|
|
404
636
|
if (gathered?.buffer) releaseBuffer(gathered.buffer);
|
|
405
637
|
if (tokenMap) releaseBuffer(tokenMap);
|
|
@@ -407,14 +639,127 @@ export async function moeFeedForwardGPU(
|
|
|
407
639
|
if (tokenOffsets) releaseBuffer(tokenOffsets);
|
|
408
640
|
if (indicesBuffer) releaseBuffer(indicesBuffer);
|
|
409
641
|
if (weightsBuffer) releaseBuffer(weightsBuffer);
|
|
642
|
+
if (ownedPerExpertScaleBuffer) releaseBuffer(ownedPerExpertScaleBuffer);
|
|
410
643
|
}
|
|
411
644
|
}
|
|
412
645
|
|
|
413
646
|
function inferBufferDtype(buffer, expectedElements) {
|
|
647
|
+
if (isWeightBuffer(buffer)) {
|
|
648
|
+
return getWeightDtype(buffer);
|
|
649
|
+
}
|
|
650
|
+
const dtype = getWeightDtype(buffer);
|
|
651
|
+
if (dtype) return dtype;
|
|
414
652
|
const bytesPerElement = Math.round(buffer.size / expectedElements);
|
|
415
653
|
return selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement });
|
|
416
654
|
}
|
|
417
655
|
|
|
656
|
+
function alignTo4(value) {
|
|
657
|
+
return Math.ceil(value / 4) * 4;
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
function resolveMatrixStorageStrideBytes(buffer, rows, cols, label) {
|
|
661
|
+
const dtype = inferBufferDtype(buffer, rows * cols, label);
|
|
662
|
+
if (dtype === 'q4k') {
|
|
663
|
+
return alignTo4(rows * Math.ceil(cols / QK_K) * Q4K_BLOCK_BYTES);
|
|
664
|
+
}
|
|
665
|
+
const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
|
|
666
|
+
return alignTo4(rows * cols * bytesPerElement);
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
async function runGemma4RouteExperts(
|
|
670
|
+
inputTensor,
|
|
671
|
+
indicesBuffer,
|
|
672
|
+
weightsBuffer,
|
|
673
|
+
layerRouter,
|
|
674
|
+
weights,
|
|
675
|
+
layerIdx,
|
|
676
|
+
numTokens,
|
|
677
|
+
topK,
|
|
678
|
+
hiddenSize,
|
|
679
|
+
intermediateSize,
|
|
680
|
+
activationDtype,
|
|
681
|
+
swigluLimit
|
|
682
|
+
) {
|
|
683
|
+
if (activationDtype !== 'f16') {
|
|
684
|
+
throw new Error(`[MoE] topk-route Gemma4 expert path requires f16 activations, got ${activationDtype}.`);
|
|
685
|
+
}
|
|
686
|
+
if (!weights?.gateUp || !weights?.down || weights.expertFormat !== 'gemma4') {
|
|
687
|
+
throw new Error(`[MoE] topk-route Gemma4 expert path missing packed weights for layer ${layerIdx}.`);
|
|
688
|
+
}
|
|
689
|
+
if (!layerRouter?.perExpertScale) {
|
|
690
|
+
throw new Error(`[MoE] topk-route Gemma4 expert path requires per-expert router scale for layer ${layerIdx}.`);
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
const device = getDevice();
|
|
694
|
+
const numRoutes = numTokens * topK;
|
|
695
|
+
const gateUpOutDim = intermediateSize * 2;
|
|
696
|
+
let gateUpOut = null;
|
|
697
|
+
let activated = null;
|
|
698
|
+
let routeDown = null;
|
|
699
|
+
let ownedPerExpertScaleBuffer = null;
|
|
700
|
+
|
|
701
|
+
try {
|
|
702
|
+
gateUpOut = await runGemma4RouteQ4MatmulF16A(
|
|
703
|
+
inputTensor,
|
|
704
|
+
indicesBuffer,
|
|
705
|
+
weights.gateUp,
|
|
706
|
+
{
|
|
707
|
+
numRoutes,
|
|
708
|
+
topK,
|
|
709
|
+
N: gateUpOutDim,
|
|
710
|
+
K: hiddenSize,
|
|
711
|
+
inputMode: 'token',
|
|
712
|
+
label: `moe_l${layerIdx}_route_gate_up`,
|
|
713
|
+
}
|
|
714
|
+
);
|
|
715
|
+
activated = await runSiLURowSplit(gateUpOut, {
|
|
716
|
+
numTokens: numRoutes,
|
|
717
|
+
dim: intermediateSize,
|
|
718
|
+
activation: 'gelu',
|
|
719
|
+
swigluLimit,
|
|
720
|
+
});
|
|
721
|
+
releaseBuffer(gateUpOut.buffer);
|
|
722
|
+
gateUpOut = null;
|
|
723
|
+
|
|
724
|
+
routeDown = await runGemma4RouteQ4MatmulF16A(
|
|
725
|
+
activated,
|
|
726
|
+
indicesBuffer,
|
|
727
|
+
weights.down,
|
|
728
|
+
{
|
|
729
|
+
numRoutes,
|
|
730
|
+
topK,
|
|
731
|
+
N: hiddenSize,
|
|
732
|
+
K: intermediateSize,
|
|
733
|
+
inputMode: 'route',
|
|
734
|
+
label: `moe_l${layerIdx}_route_down`,
|
|
735
|
+
}
|
|
736
|
+
);
|
|
737
|
+
releaseBuffer(activated.buffer);
|
|
738
|
+
activated = null;
|
|
739
|
+
|
|
740
|
+
const perExpertScale = resolvePerExpertScaleBuffer(device, layerRouter.perExpertScale);
|
|
741
|
+
ownedPerExpertScaleBuffer = perExpertScale.ownedBuffer;
|
|
742
|
+
const outputTensor = await runScatterAddRoutesF16ExpertScale(
|
|
743
|
+
routeDown,
|
|
744
|
+
indicesBuffer,
|
|
745
|
+
weightsBuffer,
|
|
746
|
+
perExpertScale.buffer,
|
|
747
|
+
numTokens,
|
|
748
|
+
hiddenSize,
|
|
749
|
+
topK,
|
|
750
|
+
{ label: `moe_l${layerIdx}_route_scatter` }
|
|
751
|
+
);
|
|
752
|
+
releaseBuffer(routeDown.buffer);
|
|
753
|
+
routeDown = null;
|
|
754
|
+
return outputTensor;
|
|
755
|
+
} finally {
|
|
756
|
+
if (gateUpOut?.buffer) releaseBuffer(gateUpOut.buffer);
|
|
757
|
+
if (activated?.buffer) releaseBuffer(activated.buffer);
|
|
758
|
+
if (routeDown?.buffer) releaseBuffer(routeDown.buffer);
|
|
759
|
+
if (ownedPerExpertScaleBuffer) releaseBuffer(ownedPerExpertScaleBuffer);
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
|
|
418
763
|
async function runGptOssExpert(
|
|
419
764
|
gathered,
|
|
420
765
|
expertOutputs,
|
|
@@ -587,6 +932,90 @@ async function runGptOssExpert(
|
|
|
587
932
|
}
|
|
588
933
|
}
|
|
589
934
|
|
|
935
|
+
async function runGemma4Expert(
|
|
936
|
+
gathered,
|
|
937
|
+
expertOutputs,
|
|
938
|
+
weights,
|
|
939
|
+
count,
|
|
940
|
+
inputOffset,
|
|
941
|
+
outputOffset,
|
|
942
|
+
hiddenSize,
|
|
943
|
+
intermediateSize,
|
|
944
|
+
activationDtype,
|
|
945
|
+
swigluLimit,
|
|
946
|
+
kernelPath
|
|
947
|
+
) {
|
|
948
|
+
const numExperts = weights.numExperts;
|
|
949
|
+
const expertIdx = weights.expertIdx;
|
|
950
|
+
if (!Number.isFinite(numExperts) || numExperts <= 0) {
|
|
951
|
+
throw new Error(`[MoE] Gemma-style expert ${expertIdx} missing numExperts.`);
|
|
952
|
+
}
|
|
953
|
+
if (!Number.isFinite(expertIdx) || expertIdx < 0) {
|
|
954
|
+
throw new Error('[MoE] Gemma-style expert missing expertIdx.');
|
|
955
|
+
}
|
|
956
|
+
if (expertIdx >= numExperts) {
|
|
957
|
+
throw new Error(`[MoE] Gemma-style expert index ${expertIdx} out of range for ${numExperts} experts.`);
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
const gateUpOutDim = intermediateSize * 2;
|
|
961
|
+
const gateUpStrideBytes = resolveMatrixStorageStrideBytes(
|
|
962
|
+
weights.gateUp,
|
|
963
|
+
gateUpOutDim,
|
|
964
|
+
hiddenSize,
|
|
965
|
+
'Gemma gate_up_proj'
|
|
966
|
+
);
|
|
967
|
+
const gateUpOffset = expertIdx * gateUpStrideBytes;
|
|
968
|
+
const downStrideBytes = resolveMatrixStorageStrideBytes(
|
|
969
|
+
weights.down,
|
|
970
|
+
hiddenSize,
|
|
971
|
+
intermediateSize,
|
|
972
|
+
'Gemma down_proj'
|
|
973
|
+
);
|
|
974
|
+
const downOffset = expertIdx * downStrideBytes;
|
|
975
|
+
|
|
976
|
+
const gateUpOut = await runMatmul(
|
|
977
|
+
gathered,
|
|
978
|
+
weights.gateUp,
|
|
979
|
+
count,
|
|
980
|
+
gateUpOutDim,
|
|
981
|
+
hiddenSize,
|
|
982
|
+
{
|
|
983
|
+
transposeB: true,
|
|
984
|
+
aOffset: inputOffset,
|
|
985
|
+
bOffset: gateUpOffset,
|
|
986
|
+
outputDtype: activationDtype,
|
|
987
|
+
role: 'moe_gate_up',
|
|
988
|
+
kernelPath,
|
|
989
|
+
}
|
|
990
|
+
);
|
|
991
|
+
|
|
992
|
+
const activated = await runSiLURowSplit(gateUpOut, {
|
|
993
|
+
numTokens: count,
|
|
994
|
+
dim: intermediateSize,
|
|
995
|
+
activation: 'gelu',
|
|
996
|
+
swigluLimit,
|
|
997
|
+
});
|
|
998
|
+
releaseBuffer(gateUpOut.buffer);
|
|
999
|
+
|
|
1000
|
+
await runMatmul(
|
|
1001
|
+
activated,
|
|
1002
|
+
weights.down,
|
|
1003
|
+
count,
|
|
1004
|
+
hiddenSize,
|
|
1005
|
+
intermediateSize,
|
|
1006
|
+
{
|
|
1007
|
+
transposeB: true,
|
|
1008
|
+
bOffset: downOffset,
|
|
1009
|
+
outputBuffer: expertOutputs,
|
|
1010
|
+
cOffset: outputOffset,
|
|
1011
|
+
outputDtype: activationDtype,
|
|
1012
|
+
role: 'moe_down',
|
|
1013
|
+
kernelPath,
|
|
1014
|
+
}
|
|
1015
|
+
);
|
|
1016
|
+
releaseBuffer(activated.buffer);
|
|
1017
|
+
}
|
|
1018
|
+
|
|
590
1019
|
async function runMixtralExpert(
|
|
591
1020
|
gathered,
|
|
592
1021
|
expertOutputs,
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* Supports multiple MoE architectures:
|
|
11
11
|
* - Mixtral-style (gate/up/down per expert)
|
|
12
12
|
* - GPT-OSS style (MXFP4 quantized fused gate_up + bias)
|
|
13
|
+
* - Gemma-style packed F16 gate_up/down experts
|
|
13
14
|
*
|
|
14
15
|
* @module inference/pipelines/text/moe-impl
|
|
15
16
|
*/
|
|
@@ -19,6 +20,7 @@ import type { MoERouter } from '../../moe-router.js';
|
|
|
19
20
|
import type { ExpertWeights } from './types.js';
|
|
20
21
|
import type { KernelPathSchema } from '../../../config/schema/index.js';
|
|
21
22
|
import type { ExecutionV1PoliciesSchema } from '../../../config/schema/execution-v1.schema.js';
|
|
23
|
+
import type { WeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
22
24
|
|
|
23
25
|
/**
|
|
24
26
|
* Clear the dequantization cache (call on model unload).
|
|
@@ -47,12 +49,16 @@ export interface MoEConfig {
|
|
|
47
49
|
modelType?: string | null;
|
|
48
50
|
hiddenSize: number;
|
|
49
51
|
intermediateSize: number;
|
|
52
|
+
rmsNormEps?: number;
|
|
53
|
+
expertIntermediateSize?: number;
|
|
50
54
|
numExperts: number;
|
|
51
55
|
moeTopK: number;
|
|
52
|
-
expertFormat: 'mixtral' | 'gpt-oss';
|
|
56
|
+
expertFormat: 'mixtral' | 'gpt-oss' | 'gemma4';
|
|
53
57
|
hiddenActivation: string;
|
|
54
58
|
swigluLimit: number | null;
|
|
55
59
|
activationDtype?: TensorDtype;
|
|
60
|
+
routerInputBuffer?: GPUBuffer | null;
|
|
61
|
+
routerInputDtype?: TensorDtype | null;
|
|
56
62
|
kernelPath?: KernelPathSchema | null;
|
|
57
63
|
executionPolicies?: ExecutionV1PoliciesSchema | null;
|
|
58
64
|
}
|
|
@@ -61,8 +67,10 @@ export interface MoEConfig {
|
|
|
61
67
|
* Expert weights with optional GPT-OSS quantized format.
|
|
62
68
|
*/
|
|
63
69
|
export interface MoEExpertWeights extends ExpertWeights {
|
|
64
|
-
expertFormat: 'mixtral' | 'gpt-oss';
|
|
70
|
+
expertFormat: 'mixtral' | 'gpt-oss' | 'gemma4';
|
|
65
71
|
numExperts?: number;
|
|
72
|
+
expertIntermediateSize?: number;
|
|
73
|
+
gateUp?: GPUBuffer;
|
|
66
74
|
gateUpBlocks?: GPUBuffer;
|
|
67
75
|
gateUpScales?: GPUBuffer;
|
|
68
76
|
gateUpBias?: GPUBuffer;
|
|
@@ -75,8 +83,10 @@ export interface MoEExpertWeights extends ExpertWeights {
|
|
|
75
83
|
* Layer router weights (for models with per-layer routers like GPT-OSS).
|
|
76
84
|
*/
|
|
77
85
|
export interface LayerRouterWeights {
|
|
78
|
-
weight: Float32Array | GPUBuffer;
|
|
86
|
+
weight: Float32Array | GPUBuffer | WeightBuffer;
|
|
79
87
|
bias: Float32Array | GPUBuffer | null;
|
|
88
|
+
scale?: Float32Array | GPUBuffer | WeightBuffer | null;
|
|
89
|
+
perExpertScale?: Float32Array | GPUBuffer | WeightBuffer | null;
|
|
80
90
|
}
|
|
81
91
|
|
|
82
92
|
/**
|
|
@@ -15,6 +15,14 @@ export function resolveMoeVendorProfile(modelType) {
|
|
|
15
15
|
if (modelType === 'mixtral') {
|
|
16
16
|
return selectRuleValue('kernels', 'moeMixtral', 'vendorQuirkProfile', { vendor });
|
|
17
17
|
}
|
|
18
|
+
if (modelType === 'gemma4' || modelType === 'diffusion_gemma') {
|
|
19
|
+
return {
|
|
20
|
+
preferVec4Dequant: false,
|
|
21
|
+
dequantTileShape: null,
|
|
22
|
+
routerWorkgroupSize: 256,
|
|
23
|
+
maxTokensPerExpertScale: 1.0,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
18
26
|
throw new Error(`[MoE] Unknown modelType "${modelType}" for vendor profile resolution.`);
|
|
19
27
|
}
|
|
20
28
|
|
|
@@ -66,7 +74,11 @@ export function validateMoeShape(config, options = {}) {
|
|
|
66
74
|
numExperts,
|
|
67
75
|
expertFormat,
|
|
68
76
|
} = config;
|
|
69
|
-
const modelType = options.modelType ?? (
|
|
77
|
+
const modelType = options.modelType ?? (
|
|
78
|
+
expertFormat === 'gpt-oss'
|
|
79
|
+
? 'gpt-oss'
|
|
80
|
+
: (expertFormat === 'gemma4' ? 'gemma4' : 'mixtral')
|
|
81
|
+
);
|
|
70
82
|
|
|
71
83
|
if (!Number.isFinite(hiddenSize) || hiddenSize <= 0) {
|
|
72
84
|
throw new Error(`[MoE] hiddenSize must be > 0, got ${hiddenSize}.`);
|