doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -17,6 +17,20 @@ import { isBufferActive, releaseBuffer } from '../../memory/buffer-pool.js';
|
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
export async function preloadShardsForExpert(ctx, layerIdx, expertIdx, options) {
|
|
20
|
+
const packedShardIndices = getPackedExpertShardIndices(ctx, layerIdx);
|
|
21
|
+
if (packedShardIndices.length > 0) {
|
|
22
|
+
for (const shardIndex of packedShardIndices) {
|
|
23
|
+
if (!ctx.shardCache.has(shardIndex)) {
|
|
24
|
+
await ctx.loadShard(shardIndex, options);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
if (isPackedExpertFormat(ctx.manifest?.moeConfig?.expertFormat)) {
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
|
|
20
34
|
// Get required shards from manifest mapping
|
|
21
35
|
const shardIndices = getShardsForExpert(layerIdx, expertIdx, ctx.manifest);
|
|
22
36
|
if (shardIndices.length === 0) {
|
|
@@ -87,7 +101,7 @@ export async function loadExpert(ctx, layerIdx, expertIdx) {
|
|
|
87
101
|
}
|
|
88
102
|
}
|
|
89
103
|
|
|
90
|
-
// Fall back to simple map for non-cached
|
|
104
|
+
// Fall back to simple map for non-cached packed experts.
|
|
91
105
|
const key = `layer_${layerIdx}_expert_${expertIdx}`;
|
|
92
106
|
if (ctx.experts.has(key)) {
|
|
93
107
|
return ctx.experts.get(key);
|
|
@@ -99,16 +113,19 @@ export async function loadExpert(ctx, layerIdx, expertIdx) {
|
|
|
99
113
|
await preloadShardsForExpert(ctx, layerIdx, expertIdx);
|
|
100
114
|
|
|
101
115
|
// Get tensor names from manifest if available (for logging/debugging)
|
|
102
|
-
const
|
|
116
|
+
const expertFormat = resolveExpertFormat(ctx);
|
|
117
|
+
const tensorNames = getExpertTensorNames(ctx, layerIdx, expertIdx, expertFormat);
|
|
103
118
|
if (tensorNames.length > 0) {
|
|
104
119
|
debugTrace.loader(`Expert ${layerIdx}_${expertIdx} tensors: ${tensorNames.length}`);
|
|
105
120
|
}
|
|
106
121
|
|
|
107
|
-
const expertFormat = resolveExpertFormat(ctx);
|
|
108
122
|
let weights;
|
|
109
123
|
if (expertFormat === 'gpt-oss') {
|
|
110
124
|
weights = await loadGptOssStyleExpert(ctx, layerIdx, expertIdx);
|
|
111
125
|
assertGptOssWeights(weights, layerIdx, expertIdx);
|
|
126
|
+
} else if (expertFormat === 'gemma4') {
|
|
127
|
+
weights = await loadGemma4StyleExpert(ctx, layerIdx, expertIdx);
|
|
128
|
+
assertGemma4Weights(weights, layerIdx, expertIdx);
|
|
112
129
|
} else {
|
|
113
130
|
weights = await loadMixtralStyleExpert(ctx, layerIdx, expertIdx);
|
|
114
131
|
assertMixtralWeights(weights, layerIdx, expertIdx);
|
|
@@ -125,7 +142,7 @@ export async function loadExpert(ctx, layerIdx, expertIdx) {
|
|
|
125
142
|
const sizeBytes = calculateExpertSize(weights);
|
|
126
143
|
ctx.expertCache.put(layerIdx, expertIdx, weights, sizeBytes);
|
|
127
144
|
} else {
|
|
128
|
-
//
|
|
145
|
+
// Packed expert formats use the simple map (shared across experts).
|
|
129
146
|
ctx.experts.set(key, weights);
|
|
130
147
|
}
|
|
131
148
|
|
|
@@ -151,6 +168,10 @@ async function loadMixtralStyleExpert(ctx, layerIdx, expertIdx) {
|
|
|
151
168
|
};
|
|
152
169
|
}
|
|
153
170
|
|
|
171
|
+
function isPackedExpertFormat(expertFormat) {
|
|
172
|
+
return expertFormat === 'gpt-oss' || expertFormat === 'gemma4';
|
|
173
|
+
}
|
|
174
|
+
|
|
154
175
|
function resolveExpertFormat(ctx) {
|
|
155
176
|
const manifest = ctx.manifest ?? {};
|
|
156
177
|
const moeConfig = manifest.moeConfig ?? null;
|
|
@@ -163,7 +184,7 @@ function resolveExpertFormat(ctx) {
|
|
|
163
184
|
}
|
|
164
185
|
|
|
165
186
|
const expertFormat = moeConfig.expertFormat;
|
|
166
|
-
if (expertFormat === 'gpt-oss' || expertFormat === 'mixtral') {
|
|
187
|
+
if (expertFormat === 'gpt-oss' || expertFormat === 'mixtral' || expertFormat === 'gemma4') {
|
|
167
188
|
return expertFormat;
|
|
168
189
|
}
|
|
169
190
|
if (expertFormat == null) {
|
|
@@ -175,6 +196,89 @@ function resolveExpertFormat(ctx) {
|
|
|
175
196
|
throw new Error(`[MoE] Manifest "${modelId}" has invalid expertFormat "${expertFormat}".`);
|
|
176
197
|
}
|
|
177
198
|
|
|
199
|
+
function getShardIndicesForLocation(location) {
|
|
200
|
+
if (!location) return [];
|
|
201
|
+
if (Array.isArray(location.spans) && location.spans.length > 0) {
|
|
202
|
+
return Array.from(new Set(location.spans.map((span) => span?.shardIndex).filter(Number.isInteger)));
|
|
203
|
+
}
|
|
204
|
+
return Number.isInteger(location.shardIndex) ? [location.shardIndex] : [];
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function getExistingTensorNames(ctx, candidates) {
|
|
208
|
+
const tensorLocations = ctx.tensorLocations;
|
|
209
|
+
if (!tensorLocations || typeof tensorLocations.get !== 'function') {
|
|
210
|
+
return [];
|
|
211
|
+
}
|
|
212
|
+
const names = [];
|
|
213
|
+
for (const candidateGroup of candidates) {
|
|
214
|
+
const found = candidateGroup.find((name) => tensorLocations.has(name));
|
|
215
|
+
if (found) {
|
|
216
|
+
names.push(found);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
return names;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
function getGemma4ExpertTensorCandidates(layerIdx) {
|
|
223
|
+
return [
|
|
224
|
+
[
|
|
225
|
+
`model.decoder.layers.${layerIdx}.experts.gate_up_proj`,
|
|
226
|
+
`model.encoder.language_model.layers.${layerIdx}.experts.gate_up_proj`,
|
|
227
|
+
`model.layers.${layerIdx}.experts.gate_up_proj`,
|
|
228
|
+
],
|
|
229
|
+
[
|
|
230
|
+
`model.decoder.layers.${layerIdx}.experts.down_proj`,
|
|
231
|
+
`model.encoder.language_model.layers.${layerIdx}.experts.down_proj`,
|
|
232
|
+
`model.layers.${layerIdx}.experts.down_proj`,
|
|
233
|
+
],
|
|
234
|
+
];
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function getGptOssExpertTensorCandidates(layerIdx) {
|
|
238
|
+
const prefix = `model.layers.${layerIdx}.mlp.experts`;
|
|
239
|
+
return [
|
|
240
|
+
[`${prefix}.gate_up_proj_blocks`],
|
|
241
|
+
[`${prefix}.gate_up_proj_scales`],
|
|
242
|
+
[`${prefix}.gate_up_proj_bias`],
|
|
243
|
+
[`${prefix}.down_proj_blocks`],
|
|
244
|
+
[`${prefix}.down_proj_scales`],
|
|
245
|
+
[`${prefix}.down_proj_bias`],
|
|
246
|
+
];
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function getPackedExpertTensorNames(ctx, layerIdx, expertFormat = ctx.manifest?.moeConfig?.expertFormat) {
|
|
250
|
+
if (expertFormat === 'gemma4') {
|
|
251
|
+
return getExistingTensorNames(ctx, getGemma4ExpertTensorCandidates(layerIdx));
|
|
252
|
+
}
|
|
253
|
+
if (expertFormat === 'gpt-oss') {
|
|
254
|
+
return getExistingTensorNames(ctx, getGptOssExpertTensorCandidates(layerIdx));
|
|
255
|
+
}
|
|
256
|
+
return [];
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function getPackedExpertShardIndices(ctx, layerIdx) {
|
|
260
|
+
const tensorNames = getPackedExpertTensorNames(ctx, layerIdx);
|
|
261
|
+
const shardIndices = new Set();
|
|
262
|
+
for (const name of tensorNames) {
|
|
263
|
+
const location = ctx.tensorLocations?.get?.(name);
|
|
264
|
+
for (const shardIndex of getShardIndicesForLocation(location)) {
|
|
265
|
+
shardIndices.add(shardIndex);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
return Array.from(shardIndices);
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
function getExpertTensorNames(ctx, layerIdx, expertIdx, expertFormat) {
|
|
272
|
+
const packedTensorNames = getPackedExpertTensorNames(ctx, layerIdx, expertFormat);
|
|
273
|
+
if (packedTensorNames.length > 0) {
|
|
274
|
+
return packedTensorNames;
|
|
275
|
+
}
|
|
276
|
+
if (isPackedExpertFormat(expertFormat) && !ctx.manifest?.groups?.[`layer.${layerIdx}.expert.${expertIdx}`]) {
|
|
277
|
+
return [];
|
|
278
|
+
}
|
|
279
|
+
return getTensorsForExpert(layerIdx, expertIdx, ctx.manifest);
|
|
280
|
+
}
|
|
281
|
+
|
|
178
282
|
function resolveGptOssNumExperts(ctx) {
|
|
179
283
|
const manifest = ctx.manifest ?? {};
|
|
180
284
|
const numExperts = manifest.moeConfig?.numExperts ?? null;
|
|
@@ -187,6 +291,34 @@ function resolveGptOssNumExperts(ctx) {
|
|
|
187
291
|
return numExperts;
|
|
188
292
|
}
|
|
189
293
|
|
|
294
|
+
function resolveGemma4NumExperts(ctx) {
|
|
295
|
+
const manifest = ctx.manifest ?? {};
|
|
296
|
+
const numExperts = manifest.moeConfig?.numExperts ?? null;
|
|
297
|
+
if (numExperts == null) {
|
|
298
|
+
const modelId = manifest.modelId ?? 'unknown';
|
|
299
|
+
throw new Error(`[MoE] Gemma-style manifest "${modelId}" missing moeConfig.numExperts`);
|
|
300
|
+
}
|
|
301
|
+
return numExperts;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
function resolveGemma4ExpertIntermediateSize(ctx) {
|
|
305
|
+
const manifest = ctx.manifest ?? {};
|
|
306
|
+
const expertIntermediateSize = manifest.moeConfig?.expertIntermediateSize ?? null;
|
|
307
|
+
if (expertIntermediateSize == null) {
|
|
308
|
+
const modelId = manifest.modelId ?? 'unknown';
|
|
309
|
+
throw new Error(`[MoE] Gemma-style manifest "${modelId}" missing moeConfig.expertIntermediateSize`);
|
|
310
|
+
}
|
|
311
|
+
return expertIntermediateSize;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
async function loadFirstTensor(ctx, names) {
|
|
315
|
+
for (const name of names) {
|
|
316
|
+
const tensor = await ctx.loadTensor(name);
|
|
317
|
+
if (tensor) return tensor;
|
|
318
|
+
}
|
|
319
|
+
return null;
|
|
320
|
+
}
|
|
321
|
+
|
|
190
322
|
function assertMixtralWeights(weights, layerIdx, expertIdx) {
|
|
191
323
|
const missing = [];
|
|
192
324
|
if (!weights.gate) missing.push('gate');
|
|
@@ -213,6 +345,43 @@ function assertGptOssWeights(weights, layerIdx, expertIdx) {
|
|
|
213
345
|
}
|
|
214
346
|
}
|
|
215
347
|
|
|
348
|
+
function assertGemma4Weights(weights, layerIdx, expertIdx) {
|
|
349
|
+
const missing = [];
|
|
350
|
+
if (!weights.gateUp) missing.push('experts.gate_up_proj');
|
|
351
|
+
if (!weights.down) missing.push('experts.down_proj');
|
|
352
|
+
if (!weights.expertIntermediateSize) missing.push('expertIntermediateSize');
|
|
353
|
+
if (missing.length > 0) {
|
|
354
|
+
throw new Error(
|
|
355
|
+
`[MoE] Gemma-style expert ${layerIdx}_${expertIdx} missing tensors: ${missing.join(', ')}`
|
|
356
|
+
);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
async function loadGemma4StyleExpert(ctx, layerIdx, expertIdx) {
|
|
361
|
+
const packedKey = `layer_${layerIdx}_gemma4_packed`;
|
|
362
|
+
let packed = ctx.experts.get(packedKey);
|
|
363
|
+
|
|
364
|
+
if (!packed) {
|
|
365
|
+
packed = {
|
|
366
|
+
expertFormat: 'gemma4',
|
|
367
|
+
numExperts: resolveGemma4NumExperts(ctx),
|
|
368
|
+
expertIntermediateSize: resolveGemma4ExpertIntermediateSize(ctx),
|
|
369
|
+
gateUp: await loadFirstTensor(ctx, getGemma4ExpertTensorCandidates(layerIdx)[0]),
|
|
370
|
+
down: await loadFirstTensor(ctx, getGemma4ExpertTensorCandidates(layerIdx)[1]),
|
|
371
|
+
};
|
|
372
|
+
ctx.experts.set(packedKey, packed);
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
return {
|
|
376
|
+
expertFormat: 'gemma4',
|
|
377
|
+
expertIdx,
|
|
378
|
+
numExperts: packed.numExperts,
|
|
379
|
+
expertIntermediateSize: packed.expertIntermediateSize,
|
|
380
|
+
gateUp: packed.gateUp,
|
|
381
|
+
down: packed.down,
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
|
|
216
385
|
|
|
217
386
|
async function loadGptOssStyleExpert(ctx, layerIdx, expertIdx) {
|
|
218
387
|
const gptOssPrefix = `model.layers.${layerIdx}.mlp.experts`;
|
|
@@ -19,6 +19,7 @@ import type {
|
|
|
19
19
|
ManifestEmbeddingProjectionSchema,
|
|
20
20
|
} from '../config/schema/index.js';
|
|
21
21
|
import type { TensorLocation } from './loader-types.js';
|
|
22
|
+
import type { DiffusionGemmaSelfConditioningWeights } from './loader-types.js';
|
|
22
23
|
|
|
23
24
|
/** Tensor loading function signature */
|
|
24
25
|
export type TensorLoader = (
|
|
@@ -55,6 +56,8 @@ export interface FinalWeightsContext {
|
|
|
55
56
|
embeddings: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
|
|
56
57
|
/** Optional embedding postprocessor declared by manifest */
|
|
57
58
|
embeddingPostprocessor?: ManifestEmbeddingPostprocessorSchema | null;
|
|
59
|
+
/** Manifest-owned DiffusionGemma self-conditioning enablement */
|
|
60
|
+
diffusionGemmaSelfConditioning?: boolean;
|
|
58
61
|
/** Manifest model type */
|
|
59
62
|
modelType?: string | null;
|
|
60
63
|
/** Whether LM head should fall back to tied embeddings */
|
|
@@ -75,6 +78,8 @@ export interface FinalWeightsResult {
|
|
|
75
78
|
lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
|
|
76
79
|
/** Optional embedding-only postprocessor weights */
|
|
77
80
|
embeddingPostprocessor: LoadedEmbeddingPostprocessor | null;
|
|
81
|
+
/** Optional DiffusionGemma decoder self-conditioning weights */
|
|
82
|
+
diffusionGemmaSelfConditioning: DiffusionGemmaSelfConditioningWeights | null;
|
|
78
83
|
/** Whether norm offset debug was logged */
|
|
79
84
|
normOffsetDebugLogged: boolean;
|
|
80
85
|
}
|
|
@@ -5,6 +5,8 @@ import {
|
|
|
5
5
|
createCpuWeightBuffer,
|
|
6
6
|
isWeightBuffer,
|
|
7
7
|
isCpuWeightBuffer,
|
|
8
|
+
isSplitWeightBuffer,
|
|
9
|
+
getWeightDtype,
|
|
8
10
|
} from '../gpu/weight-buffer.js';
|
|
9
11
|
import { maybeDowncastToF16 } from './weight-downcast.js';
|
|
10
12
|
import { getTensorNamesByRole } from './tensors/tensor-role.js';
|
|
@@ -21,6 +23,12 @@ const HEAD_GROUP = 'head';
|
|
|
21
23
|
const FINAL_NORM_ROLE = 'norm';
|
|
22
24
|
const LM_HEAD_ROLE = 'lm_head';
|
|
23
25
|
const EMBEDDING_MODEL_TYPE = 'embedding';
|
|
26
|
+
const DENSE_TIED_LM_HEAD_DTYPES = new Set(['F16', 'F32']);
|
|
27
|
+
const DIFFUSION_GEMMA_SELF_CONDITIONING_PREFIXES = [
|
|
28
|
+
'model.decoder.self_conditioning',
|
|
29
|
+
'decoder.self_conditioning',
|
|
30
|
+
'self_conditioning',
|
|
31
|
+
];
|
|
24
32
|
|
|
25
33
|
function isGpuBufferInstance(value) {
|
|
26
34
|
return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
|
|
@@ -54,6 +62,77 @@ function createRangeBackedWeightBuffer(ctx, name, location) {
|
|
|
54
62
|
return createCpuWeightBuffer(source, dtype, layout, location.shape, name);
|
|
55
63
|
}
|
|
56
64
|
|
|
65
|
+
function normalizeLocationDtype(location) {
|
|
66
|
+
return typeof location?.dtype === 'string'
|
|
67
|
+
? location.dtype.trim().toUpperCase()
|
|
68
|
+
: '';
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function isDenseTiedLmHeadLocation(location) {
|
|
72
|
+
if (!location || !DENSE_TIED_LM_HEAD_DTYPES.has(normalizeLocationDtype(location))) {
|
|
73
|
+
return false;
|
|
74
|
+
}
|
|
75
|
+
if (location.sourceTransform || location.storage) {
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
return true;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function getLoadedWeightShape(weight) {
|
|
82
|
+
if (isWeightBuffer(weight) || isCpuWeightBuffer(weight) || isSplitWeightBuffer(weight)) {
|
|
83
|
+
return weight.shape;
|
|
84
|
+
}
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function getLoadedWeightLayout(weight) {
|
|
89
|
+
if (isWeightBuffer(weight) || isCpuWeightBuffer(weight) || isSplitWeightBuffer(weight)) {
|
|
90
|
+
return weight.layout;
|
|
91
|
+
}
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function shapesEqual(left, right) {
|
|
96
|
+
if (!Array.isArray(left) || !Array.isArray(right) || left.length !== right.length) {
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
for (let i = 0; i < left.length; i++) {
|
|
100
|
+
if (left[i] !== right[i]) {
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return true;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function shouldUseTiedEmbeddingsForLmHead(ctx, loc) {
|
|
108
|
+
if (ctx.tieWordEmbeddings !== true || !ctx.embeddings) {
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
if (ctx.embeddings instanceof Float32Array || isCpuWeightBuffer(ctx.embeddings)) {
|
|
112
|
+
return false;
|
|
113
|
+
}
|
|
114
|
+
if (!isDenseTiedLmHeadLocation(loc)) {
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const embeddingShape = getLoadedWeightShape(ctx.embeddings);
|
|
119
|
+
if (!shapesEqual(loc.shape, embeddingShape)) {
|
|
120
|
+
return false;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const lmHeadDtype = selectRuleValue('loader', 'weights', 'floatLocationDtype', {
|
|
124
|
+
locationDtype: loc.dtype,
|
|
125
|
+
});
|
|
126
|
+
const embeddingDtype = getWeightDtype(ctx.embeddings);
|
|
127
|
+
if (embeddingDtype !== lmHeadDtype) {
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const lmHeadLayout = ctx.resolveWeightLayout(loc);
|
|
132
|
+
const embeddingLayout = getLoadedWeightLayout(ctx.embeddings);
|
|
133
|
+
return embeddingLayout === lmHeadLayout;
|
|
134
|
+
}
|
|
135
|
+
|
|
57
136
|
function isLikelyFinalNormName(name) {
|
|
58
137
|
const lower = String(name || '').toLowerCase();
|
|
59
138
|
if (!lower) return false;
|
|
@@ -65,6 +144,8 @@ function isLikelyFinalNormName(name) {
|
|
|
65
144
|
|
|
66
145
|
return (
|
|
67
146
|
lower === 'norm.weight' ||
|
|
147
|
+
lower === 'model.decoder.norm.weight' ||
|
|
148
|
+
lower === 'model.encoder.language_model.norm.weight' ||
|
|
68
149
|
lower.includes('model.norm.weight') ||
|
|
69
150
|
lower.includes('language_model.norm.weight') ||
|
|
70
151
|
lower.includes('model.language_model.norm.weight') ||
|
|
@@ -76,6 +157,71 @@ function isLikelyFinalNormName(name) {
|
|
|
76
157
|
);
|
|
77
158
|
}
|
|
78
159
|
|
|
160
|
+
async function loadFirstExistingTensor(ctx, prefixes, suffixes, options = {}) {
|
|
161
|
+
for (const prefix of prefixes) {
|
|
162
|
+
for (const suffix of suffixes) {
|
|
163
|
+
const tensor = await ctx.loadTensor(`${prefix}.${suffix}`, options.toGPU !== false, true);
|
|
164
|
+
if (tensor) return tensor;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
async function loadDiffusionGemmaSelfConditioning(ctx) {
|
|
171
|
+
const preNorm = await loadFirstExistingTensor(
|
|
172
|
+
ctx,
|
|
173
|
+
DIFFUSION_GEMMA_SELF_CONDITIONING_PREFIXES,
|
|
174
|
+
['pre_norm.weight']
|
|
175
|
+
);
|
|
176
|
+
const gateProj = await loadFirstExistingTensor(
|
|
177
|
+
ctx,
|
|
178
|
+
DIFFUSION_GEMMA_SELF_CONDITIONING_PREFIXES,
|
|
179
|
+
['gate_proj.weight']
|
|
180
|
+
);
|
|
181
|
+
const upProj = await loadFirstExistingTensor(
|
|
182
|
+
ctx,
|
|
183
|
+
DIFFUSION_GEMMA_SELF_CONDITIONING_PREFIXES,
|
|
184
|
+
['up_proj.weight']
|
|
185
|
+
);
|
|
186
|
+
const downProj = await loadFirstExistingTensor(
|
|
187
|
+
ctx,
|
|
188
|
+
DIFFUSION_GEMMA_SELF_CONDITIONING_PREFIXES,
|
|
189
|
+
['down_proj.weight']
|
|
190
|
+
);
|
|
191
|
+
const postNorm = await loadFirstExistingTensor(
|
|
192
|
+
ctx,
|
|
193
|
+
DIFFUSION_GEMMA_SELF_CONDITIONING_PREFIXES,
|
|
194
|
+
['post_norm.weight']
|
|
195
|
+
);
|
|
196
|
+
|
|
197
|
+
const anyPresent = preNorm || gateProj || upProj || downProj || postNorm;
|
|
198
|
+
if (!anyPresent) {
|
|
199
|
+
return null;
|
|
200
|
+
}
|
|
201
|
+
const missing = [];
|
|
202
|
+
if (!preNorm) missing.push('pre_norm.weight');
|
|
203
|
+
if (!gateProj) missing.push('gate_proj.weight');
|
|
204
|
+
if (!upProj) missing.push('up_proj.weight');
|
|
205
|
+
if (!downProj) missing.push('down_proj.weight');
|
|
206
|
+
if (missing.length > 0) {
|
|
207
|
+
throw new Error(
|
|
208
|
+
`[Loader] DiffusionGemma self-conditioning weights are incomplete. Missing: ${missing.join(', ')}.`
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
return {
|
|
213
|
+
preNorm,
|
|
214
|
+
postNorm,
|
|
215
|
+
gateProj,
|
|
216
|
+
upProj,
|
|
217
|
+
downProj,
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function shouldLoadDiffusionGemmaSelfConditioning(ctx) {
|
|
222
|
+
return ctx.diffusionGemmaSelfConditioning === true;
|
|
223
|
+
}
|
|
224
|
+
|
|
79
225
|
// ============================================================================
|
|
80
226
|
// Main Function
|
|
81
227
|
// ============================================================================
|
|
@@ -93,11 +239,15 @@ export async function loadFinalWeights(ctx) {
|
|
|
93
239
|
// Load LM head
|
|
94
240
|
const lmHead = await loadLmHead(ctx);
|
|
95
241
|
const embeddingPostprocessor = await loadEmbeddingPostprocessor(ctx);
|
|
242
|
+
const diffusionGemmaSelfConditioning = shouldLoadDiffusionGemmaSelfConditioning(ctx)
|
|
243
|
+
? await loadDiffusionGemmaSelfConditioning(ctx)
|
|
244
|
+
: null;
|
|
96
245
|
|
|
97
246
|
return {
|
|
98
247
|
finalNorm,
|
|
99
248
|
lmHead,
|
|
100
249
|
embeddingPostprocessor,
|
|
250
|
+
diffusionGemmaSelfConditioning,
|
|
101
251
|
normOffsetDebugLogged,
|
|
102
252
|
};
|
|
103
253
|
}
|
|
@@ -185,6 +335,14 @@ async function loadLmHead(ctx) {
|
|
|
185
335
|
const loc = ctx.tensorLocations.get(name);
|
|
186
336
|
if (!loc) continue;
|
|
187
337
|
|
|
338
|
+
if (shouldUseTiedEmbeddingsForLmHead(ctx, loc)) {
|
|
339
|
+
debugTrace.loader(`Using tied embeddings as dense LM head "${name}" (manifest.tieWordEmbeddings=true)`);
|
|
340
|
+
lmHeadName = name;
|
|
341
|
+
lmHeadLoc = loc;
|
|
342
|
+
lmHead = ctx.embeddings;
|
|
343
|
+
break;
|
|
344
|
+
}
|
|
345
|
+
|
|
188
346
|
const shouldStream = ctx.shouldStreamLargeWeight(name, loc, 'LM head');
|
|
189
347
|
const tensor = shouldStream
|
|
190
348
|
? (
|
|
@@ -192,17 +350,18 @@ async function loadLmHead(ctx) {
|
|
|
192
350
|
?? await ctx.loadTensor(name, false, true)
|
|
193
351
|
)
|
|
194
352
|
: await ctx.loadTensor(name, true, true);
|
|
353
|
+
const tensorShouldStream = shouldStream;
|
|
195
354
|
|
|
196
|
-
if (
|
|
355
|
+
if (tensorShouldStream && tensor && !(tensor instanceof Float32Array) && !isCpuWeightBuffer(tensor)) {
|
|
197
356
|
throw new Error(
|
|
198
357
|
`[Loader] LM head "${name}" too large for GPU and cannot be loaded on CPU (dtype=${loc.dtype}).`
|
|
199
358
|
);
|
|
200
359
|
}
|
|
201
360
|
|
|
202
|
-
if (tensor && (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || isCpuWeightBuffer(tensor) || tensor instanceof Float32Array)) {
|
|
361
|
+
if (tensor && (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || isCpuWeightBuffer(tensor) || isSplitWeightBuffer(tensor) || tensor instanceof Float32Array)) {
|
|
203
362
|
lmHeadName = name;
|
|
204
363
|
lmHeadLoc = loc;
|
|
205
|
-
lmHead = processLmHeadTensor(ctx, tensor, name, loc,
|
|
364
|
+
lmHead = processLmHeadTensor(ctx, tensor, name, loc, tensorShouldStream);
|
|
206
365
|
break;
|
|
207
366
|
}
|
|
208
367
|
}
|
|
@@ -287,6 +446,10 @@ async function loadCpuFloatTensor(ctx, tensorName, label) {
|
|
|
287
446
|
|
|
288
447
|
|
|
289
448
|
function processLmHeadTensor(ctx, tensor, name, loc, shouldStream) {
|
|
449
|
+
if (isSplitWeightBuffer(tensor)) {
|
|
450
|
+
return tensor;
|
|
451
|
+
}
|
|
452
|
+
|
|
290
453
|
if (isCpuWeightBuffer(tensor)) {
|
|
291
454
|
log.warn('Loader', `LM head stored on CPU via range-backed source (layout=${tensor.layout})`);
|
|
292
455
|
return tensor;
|
|
@@ -332,7 +495,7 @@ async function maybeDowncastLmHead(ctx, lmHead, lmHeadName, lmHeadLoc) {
|
|
|
332
495
|
}
|
|
333
496
|
|
|
334
497
|
// Can't downcast Float32Array or CpuWeightBuffer
|
|
335
|
-
if (lmHead instanceof Float32Array || isCpuWeightBuffer(lmHead)) {
|
|
498
|
+
if (lmHead instanceof Float32Array || isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead)) {
|
|
336
499
|
return lmHead;
|
|
337
500
|
}
|
|
338
501
|
|
|
@@ -38,6 +38,8 @@ export interface LayerLoaderContext {
|
|
|
38
38
|
isMoE: boolean;
|
|
39
39
|
/** Check if layer is an expert layer */
|
|
40
40
|
isExpertLayer: (layerIdx: number) => boolean;
|
|
41
|
+
/** Load dense FFN weights even when a layer also has MoE experts. */
|
|
42
|
+
loadDenseFfnForMoeLayers?: boolean;
|
|
41
43
|
/** Optional attention dimensions for fused QKV sizing */
|
|
42
44
|
numHeads?: number;
|
|
43
45
|
numKVHeads?: number;
|
|
@@ -22,6 +22,8 @@ import { dequantizeQ4KM, dequantizeQ4KMRowWise } from '../converter/quantizer.js
|
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
const LAYER_PREFIXES = (layerIdx) => [
|
|
25
|
+
`model.decoder.layers.${layerIdx}`,
|
|
26
|
+
`model.encoder.language_model.layers.${layerIdx}`,
|
|
25
27
|
`model.language_model.layers.${layerIdx}`,
|
|
26
28
|
`language_model.layers.${layerIdx}`,
|
|
27
29
|
`language_model.model.layers.${layerIdx}`,
|
|
@@ -41,7 +43,10 @@ const ATTN_SUFFIXES = {
|
|
|
41
43
|
kNorm: ['self_attn.k_norm.weight', 'self_attn.k_layernorm.weight', 'attn_k_norm.weight'],
|
|
42
44
|
postAttentionNorm: ['post_attention_layernorm.weight', 'post_attention_norm.weight', 'ffn_norm.weight'],
|
|
43
45
|
preFeedforwardNorm: ['pre_feedforward_layernorm.weight'],
|
|
46
|
+
preFeedforwardNorm2: ['pre_feedforward_layernorm_2.weight'],
|
|
44
47
|
postFeedforwardNorm: ['post_feedforward_layernorm.weight', 'post_ffw_norm.weight'],
|
|
48
|
+
postFeedforwardNorm1: ['post_feedforward_layernorm_1.weight'],
|
|
49
|
+
postFeedforwardNorm2: ['post_feedforward_layernorm_2.weight'],
|
|
45
50
|
postPerLayerInputNorm: ['post_per_layer_input_norm.weight'],
|
|
46
51
|
layerScalar: ['layer_scalar'],
|
|
47
52
|
};
|
|
@@ -76,8 +81,10 @@ const FFN_SUFFIXES = {
|
|
|
76
81
|
|
|
77
82
|
|
|
78
83
|
const ROUTER_SUFFIXES = {
|
|
79
|
-
routerWeight: ['mlp.router.weight', 'block_sparse_moe.gate.weight'],
|
|
84
|
+
routerWeight: ['mlp.router.weight', 'block_sparse_moe.gate.weight', 'router.proj.weight'],
|
|
80
85
|
routerBias: ['mlp.router.bias'],
|
|
86
|
+
routerScale: ['router.scale'],
|
|
87
|
+
routerPerExpertScale: ['router.per_expert_scale'],
|
|
81
88
|
};
|
|
82
89
|
|
|
83
90
|
|
|
@@ -345,7 +352,10 @@ export async function loadLayer(ctx, layerIdx) {
|
|
|
345
352
|
kNorm: null,
|
|
346
353
|
postAttentionNorm: null,
|
|
347
354
|
preFeedforwardNorm: null,
|
|
355
|
+
preFeedforwardNorm2: null,
|
|
348
356
|
postFeedforwardNorm: null,
|
|
357
|
+
postFeedforwardNorm1: null,
|
|
358
|
+
postFeedforwardNorm2: null,
|
|
349
359
|
postNorm: null,
|
|
350
360
|
postAttnNorm: null,
|
|
351
361
|
convInProj: null,
|
|
@@ -368,14 +378,14 @@ export async function loadLayer(ctx, layerIdx) {
|
|
|
368
378
|
// Load attention weights in parallel
|
|
369
379
|
await loadAttentionWeights(ctx, weights, layerIdx, tryLoad, tryLoadNorm);
|
|
370
380
|
|
|
371
|
-
// Load FFN weights (unless MoE expert layer)
|
|
372
|
-
if (!ctx.isMoE || !ctx.isExpertLayer(layerIdx)) {
|
|
381
|
+
// Load FFN weights (unless MoE expert layer owns only routed experts).
|
|
382
|
+
if (!ctx.isMoE || !ctx.isExpertLayer(layerIdx) || ctx.loadDenseFfnForMoeLayers === true) {
|
|
373
383
|
await loadFfnWeights(ctx, weights, layerIdx, tryLoad, prefixes);
|
|
374
384
|
}
|
|
375
385
|
|
|
376
386
|
// Load MoE router weights
|
|
377
387
|
if (ctx.isMoE && ctx.isExpertLayer(layerIdx)) {
|
|
378
|
-
await loadRouterWeights(weights, tryLoad);
|
|
388
|
+
await loadRouterWeights(ctx, weights, layerIdx, tryLoad);
|
|
379
389
|
}
|
|
380
390
|
|
|
381
391
|
// Load attention sinks
|
|
@@ -448,7 +458,10 @@ async function loadAttentionWeights(ctx, weights, layerIdx, tryLoad, tryLoadNorm
|
|
|
448
458
|
kNorm,
|
|
449
459
|
postAttentionNorm,
|
|
450
460
|
preFeedforwardNorm,
|
|
461
|
+
preFeedforwardNorm2,
|
|
451
462
|
postFeedforwardNorm,
|
|
463
|
+
postFeedforwardNorm1,
|
|
464
|
+
postFeedforwardNorm2,
|
|
452
465
|
postPerLayerInputNorm,
|
|
453
466
|
layerScalar,
|
|
454
467
|
convInProj,
|
|
@@ -474,7 +487,10 @@ async function loadAttentionWeights(ctx, weights, layerIdx, tryLoad, tryLoadNorm
|
|
|
474
487
|
tryLoadNorm(ATTN_SUFFIXES.kNorm),
|
|
475
488
|
tryLoadNorm(ATTN_SUFFIXES.postAttentionNorm),
|
|
476
489
|
tryLoadNorm(ATTN_SUFFIXES.preFeedforwardNorm),
|
|
490
|
+
tryLoadNorm(ATTN_SUFFIXES.preFeedforwardNorm2),
|
|
477
491
|
tryLoadNorm(ATTN_SUFFIXES.postFeedforwardNorm),
|
|
492
|
+
tryLoadNorm(ATTN_SUFFIXES.postFeedforwardNorm1),
|
|
493
|
+
tryLoadNorm(ATTN_SUFFIXES.postFeedforwardNorm2),
|
|
478
494
|
tryLoadNorm(ATTN_SUFFIXES.postPerLayerInputNorm),
|
|
479
495
|
tryLoadCpu(ATTN_SUFFIXES.layerScalar),
|
|
480
496
|
tryLoad(CONV_SUFFIXES.convInProj),
|
|
@@ -510,7 +526,10 @@ async function loadAttentionWeights(ctx, weights, layerIdx, tryLoad, tryLoadNorm
|
|
|
510
526
|
|
|
511
527
|
weights.postAttentionNorm = postAttentionNorm;
|
|
512
528
|
weights.preFeedforwardNorm = preFeedforwardNorm;
|
|
529
|
+
weights.preFeedforwardNorm2 = preFeedforwardNorm2;
|
|
513
530
|
weights.postFeedforwardNorm = postFeedforwardNorm;
|
|
531
|
+
weights.postFeedforwardNorm1 = postFeedforwardNorm1;
|
|
532
|
+
weights.postFeedforwardNorm2 = postFeedforwardNorm2;
|
|
514
533
|
weights.postPerLayerInputNorm = postPerLayerInputNorm;
|
|
515
534
|
weights.layerScalar = layerScalar;
|
|
516
535
|
weights.postNorm = weights.postAttentionNorm || weights.preFeedforwardNorm;
|
|
@@ -607,14 +626,63 @@ async function loadStablePerLayerInputGate(ctx, layerIdx, prefixes, tryLoad) {
|
|
|
607
626
|
}
|
|
608
627
|
|
|
609
628
|
|
|
610
|
-
|
|
611
|
-
|
|
629
|
+
function getVectorElementCount(shape, label) {
|
|
630
|
+
if (!Array.isArray(shape) || shape.length === 0) {
|
|
631
|
+
throw new Error(`[LayerLoader] ${label} requires a non-empty shape.`);
|
|
632
|
+
}
|
|
633
|
+
return shape.reduce((product, dim) => {
|
|
634
|
+
if (!Number.isInteger(dim) || dim <= 0) {
|
|
635
|
+
throw new Error(`[LayerLoader] ${label} has invalid shape dimension: ${String(dim)}.`);
|
|
636
|
+
}
|
|
637
|
+
return product * dim;
|
|
638
|
+
}, 1);
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
async function materializeRouterPerExpertScale(ctx, weight, layerIdx) {
|
|
642
|
+
if (!weight || !isWeightBuffer(weight)) return weight;
|
|
643
|
+
const dtype = getWeightDtype(weight);
|
|
644
|
+
if (dtype === 'f32') return weight;
|
|
645
|
+
if (dtype !== 'f16') {
|
|
646
|
+
throw new Error(
|
|
647
|
+
`[LayerLoader] Layer ${layerIdx} router per-expert scale must be f16 or f32, got ${String(dtype)}.`
|
|
648
|
+
);
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
const shape = [...weight.shape];
|
|
652
|
+
const numElements = getVectorElementCount(shape, 'router per-expert scale');
|
|
653
|
+
const inputTensor = createTensor(
|
|
654
|
+
weight.buffer,
|
|
655
|
+
'f16',
|
|
656
|
+
[numElements],
|
|
657
|
+
`${weight.label ?? `layer_${layerIdx}_router_per_expert_scale`}_f16`
|
|
658
|
+
);
|
|
659
|
+
const f32Tensor = await castF16ToF32(inputTensor);
|
|
660
|
+
ctx.gpuBuffers.add(f32Tensor.buffer);
|
|
661
|
+
debugTrace.loader(`Layer ${layerIdx} materialized router per-expert scale F16->F32`);
|
|
662
|
+
|
|
663
|
+
return createWeightBuffer(
|
|
664
|
+
f32Tensor.buffer,
|
|
665
|
+
'f32',
|
|
666
|
+
weight.layout,
|
|
667
|
+
shape,
|
|
668
|
+
weight.label ?? `layer_${layerIdx}_router_per_expert_scale`,
|
|
669
|
+
cloneWeightMaterializations(weight, ['f32']),
|
|
670
|
+
weight.metadata ?? null
|
|
671
|
+
);
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
async function loadRouterWeights(ctx, weights, layerIdx, tryLoad) {
|
|
675
|
+
const [routerWeight, routerBias, routerScale, routerPerExpertScale] = await Promise.all([
|
|
612
676
|
tryLoad(ROUTER_SUFFIXES.routerWeight),
|
|
613
677
|
tryLoad(ROUTER_SUFFIXES.routerBias),
|
|
678
|
+
tryLoad(ROUTER_SUFFIXES.routerScale),
|
|
679
|
+
tryLoad(ROUTER_SUFFIXES.routerPerExpertScale),
|
|
614
680
|
]);
|
|
615
681
|
// Router weights follow matmul dtype/layout rules when present
|
|
616
682
|
weights.routerWeight = (routerWeight);
|
|
617
683
|
weights.routerBias = (routerBias);
|
|
684
|
+
weights.routerScale = (routerScale);
|
|
685
|
+
weights.routerPerExpertScale = (await materializeRouterPerExpertScale(ctx, routerPerExpertScale, layerIdx));
|
|
618
686
|
}
|
|
619
687
|
|
|
620
688
|
// ============================================================================
|