doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -9,8 +9,9 @@ import { f16ToF32, convertBF16ToF32GPU, shouldDequantizeToF16, applyBufferLayout
|
|
|
9
9
|
import { QK_K, Q4K_BLOCK_BYTES, Q6K_BLOCK_BYTES } from '../quantization-constants.js';
|
|
10
10
|
import { log, trace as debugTrace } from '../../debug/index.js';
|
|
11
11
|
import { selectRuleValue } from '../../rules/rule-registry.js';
|
|
12
|
-
import { dequantizeQ4KM, dequantizeQ4KMRowWise } from '../../converter/quantizer.js';
|
|
12
|
+
import { dequantizeQ4KM, dequantizeQ4KMRowWise, float32ToFloat16 } from '../../converter/quantizer.js';
|
|
13
13
|
import { hasSourceTransform } from './source-transform.js';
|
|
14
|
+
import { loadFunctionalDescriptor } from './functional-descriptor-loader.js';
|
|
14
15
|
|
|
15
16
|
// ============================================================================
|
|
16
17
|
// Q4K Detection
|
|
@@ -77,6 +78,28 @@ function toUint8View(data) {
|
|
|
77
78
|
return new Uint8Array(data);
|
|
78
79
|
}
|
|
79
80
|
|
|
81
|
+
function toUint16View(data, label) {
|
|
82
|
+
const bytes = toUint8View(data);
|
|
83
|
+
if (bytes.byteLength % 2 !== 0) {
|
|
84
|
+
throw new Error(`${label}: byte length must be divisible by 2.`);
|
|
85
|
+
}
|
|
86
|
+
if (bytes.byteOffset % 2 === 0) {
|
|
87
|
+
return new Uint16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2);
|
|
88
|
+
}
|
|
89
|
+
return new Uint16Array(bytes.slice().buffer);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function toFloat32View(data, label) {
|
|
93
|
+
const bytes = toUint8View(data);
|
|
94
|
+
if (bytes.byteLength % 4 !== 0) {
|
|
95
|
+
throw new Error(`${label}: byte length must be divisible by 4.`);
|
|
96
|
+
}
|
|
97
|
+
if (bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength) {
|
|
98
|
+
return new Float32Array(bytes.buffer);
|
|
99
|
+
}
|
|
100
|
+
return new Float32Array(bytes.slice().buffer);
|
|
101
|
+
}
|
|
102
|
+
|
|
80
103
|
function resolveInputByteLength(data, fallbackSize) {
|
|
81
104
|
if (data instanceof Uint8Array) return data.byteLength;
|
|
82
105
|
if (ArrayBuffer.isView(data)) return data.byteLength;
|
|
@@ -108,12 +131,203 @@ function getShapeElementCount(shape) {
|
|
|
108
131
|
return shape.reduce((product, value) => product * value, 1);
|
|
109
132
|
}
|
|
110
133
|
|
|
134
|
+
function getStorageCompanion(shardData, location, name, role) {
|
|
135
|
+
const companion = shardData?.storageCompanions?.[role];
|
|
136
|
+
if (!companion || !(companion.bytes instanceof Uint8Array)) {
|
|
137
|
+
throw new Error(
|
|
138
|
+
`W4A16 tensor "${name}" is missing required storage companion "${role}".`
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
const declared = Array.isArray(location?.storage?.companions)
|
|
142
|
+
? location.storage.companions.find((entry) => entry.role === role)
|
|
143
|
+
: null;
|
|
144
|
+
if (declared && companion.tensorId !== declared.tensorId) {
|
|
145
|
+
throw new Error(
|
|
146
|
+
`W4A16 tensor "${name}" companion "${role}" resolved to "${companion.tensorId}", expected "${declared.tensorId}".`
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
return companion;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function readW4A16LogicalShape(companion, fallbackShape, name) {
|
|
153
|
+
const location = companion.location ?? null;
|
|
154
|
+
const bytes = companion.bytes;
|
|
155
|
+
const dtype = String(location?.dtype || '').toUpperCase();
|
|
156
|
+
if (!Array.isArray(location?.shape) || location.shape.length !== 1 || location.shape[0] !== 2) {
|
|
157
|
+
throw new Error(`W4A16 tensor "${name}" shape companion must have shape [2].`);
|
|
158
|
+
}
|
|
159
|
+
if (dtype === 'I64') {
|
|
160
|
+
if (bytes.byteLength !== 16) {
|
|
161
|
+
throw new Error(`W4A16 tensor "${name}" I64 shape companion must be 16 bytes.`);
|
|
162
|
+
}
|
|
163
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
164
|
+
return [Number(view.getBigInt64(0, true)), Number(view.getBigInt64(8, true))];
|
|
165
|
+
}
|
|
166
|
+
if (dtype === 'I32' || dtype === 'U32') {
|
|
167
|
+
if (bytes.byteLength !== 8) {
|
|
168
|
+
throw new Error(`W4A16 tensor "${name}" ${dtype} shape companion must be 8 bytes.`);
|
|
169
|
+
}
|
|
170
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
171
|
+
const read = dtype === 'I32'
|
|
172
|
+
? (offset) => view.getInt32(offset, true)
|
|
173
|
+
: (offset) => view.getUint32(offset, true);
|
|
174
|
+
return [read(0), read(4)];
|
|
175
|
+
}
|
|
176
|
+
if (Array.isArray(fallbackShape) && fallbackShape.length === 2) {
|
|
177
|
+
return fallbackShape;
|
|
178
|
+
}
|
|
179
|
+
throw new Error(`W4A16 tensor "${name}" has unsupported shape companion dtype "${location?.dtype}".`);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function assertW4A16Shape(shape, fallbackShape, name) {
|
|
183
|
+
if (!Array.isArray(shape) || shape.length !== 2) {
|
|
184
|
+
throw new Error(`W4A16 tensor "${name}" logical shape must be 2D.`);
|
|
185
|
+
}
|
|
186
|
+
const rows = Number(shape[0]);
|
|
187
|
+
const cols = Number(shape[1]);
|
|
188
|
+
if (!Number.isInteger(rows) || rows <= 0 || !Number.isInteger(cols) || cols <= 0) {
|
|
189
|
+
throw new Error(`W4A16 tensor "${name}" has invalid logical shape ${JSON.stringify(shape)}.`);
|
|
190
|
+
}
|
|
191
|
+
if (Array.isArray(fallbackShape) && fallbackShape.length === 2) {
|
|
192
|
+
if (rows !== fallbackShape[0] || cols !== fallbackShape[1]) {
|
|
193
|
+
throw new Error(
|
|
194
|
+
`W4A16 tensor "${name}" shape companion [${rows},${cols}] does not match manifest shape [${fallbackShape.join(',')}].`
|
|
195
|
+
);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
return [rows, cols];
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function bf16ToF32(bits) {
|
|
202
|
+
const floats = new Float32Array(1);
|
|
203
|
+
const uints = new Uint32Array(floats.buffer);
|
|
204
|
+
uints[0] = (bits & 0xffff) << 16;
|
|
205
|
+
return floats[0];
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function readOffsetBinaryInt4(byte, highNibble) {
|
|
209
|
+
const value = highNibble ? ((byte >> 4) & 0x0f) : (byte & 0x0f);
|
|
210
|
+
return value - 8;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function readW4A16Scales(companion, expectedScales, name) {
|
|
214
|
+
const dtype = String(companion.location?.dtype || '').toUpperCase();
|
|
215
|
+
if (dtype === 'F16') {
|
|
216
|
+
const packed = toUint16View(companion.bytes, `W4A16 scales for ${name}`);
|
|
217
|
+
if (packed.length !== expectedScales) {
|
|
218
|
+
throw new Error(
|
|
219
|
+
`W4A16 tensor "${name}" scale count ${packed.length} does not match expected ${expectedScales}.`
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
const scales = new Float32Array(packed.length);
|
|
223
|
+
for (let i = 0; i < packed.length; i += 1) {
|
|
224
|
+
scales[i] = f16ToF32(packed[i]);
|
|
225
|
+
}
|
|
226
|
+
return scales;
|
|
227
|
+
}
|
|
228
|
+
if (dtype === 'BF16') {
|
|
229
|
+
const packed = toUint16View(companion.bytes, `W4A16 scales for ${name}`);
|
|
230
|
+
if (packed.length !== expectedScales) {
|
|
231
|
+
throw new Error(
|
|
232
|
+
`W4A16 tensor "${name}" scale count ${packed.length} does not match expected ${expectedScales}.`
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
const scales = new Float32Array(packed.length);
|
|
236
|
+
for (let i = 0; i < packed.length; i += 1) {
|
|
237
|
+
scales[i] = bf16ToF32(packed[i]);
|
|
238
|
+
}
|
|
239
|
+
return scales;
|
|
240
|
+
}
|
|
241
|
+
if (dtype === 'F32') {
|
|
242
|
+
const scales = toFloat32View(companion.bytes, `W4A16 scales for ${name}`);
|
|
243
|
+
if (scales.length !== expectedScales) {
|
|
244
|
+
throw new Error(
|
|
245
|
+
`W4A16 tensor "${name}" scale count ${scales.length} does not match expected ${expectedScales}.`
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
return scales;
|
|
249
|
+
}
|
|
250
|
+
throw new Error(`W4A16 tensor "${name}" has unsupported scale companion dtype "${companion.location?.dtype}".`);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function validateW4A16PackedStorage(shardData, location, name) {
|
|
254
|
+
const scaleCompanion = getStorageCompanion(shardData, location, name, 'scales');
|
|
255
|
+
const shapeCompanion = getStorageCompanion(shardData, location, name, 'shape');
|
|
256
|
+
const [rows, cols] = assertW4A16Shape(
|
|
257
|
+
readW4A16LogicalShape(shapeCompanion, location.shape, name),
|
|
258
|
+
location.shape,
|
|
259
|
+
name
|
|
260
|
+
);
|
|
261
|
+
const groupsPerRow = Math.ceil(cols / 32);
|
|
262
|
+
const expectedPackedBytes = rows * groupsPerRow * 16;
|
|
263
|
+
if (shardData.byteLength !== expectedPackedBytes) {
|
|
264
|
+
throw new Error(
|
|
265
|
+
`W4A16 tensor "${name}" packed byte length ${shardData.byteLength} does not match expected ${expectedPackedBytes}.`
|
|
266
|
+
);
|
|
267
|
+
}
|
|
268
|
+
const scaleDtype = String(scaleCompanion.location?.dtype || '').toUpperCase();
|
|
269
|
+
const scaleBytesPerElement = scaleDtype === 'F32'
|
|
270
|
+
? 4
|
|
271
|
+
: (scaleDtype === 'F16' || scaleDtype === 'BF16' ? 2 : null);
|
|
272
|
+
if (scaleBytesPerElement == null) {
|
|
273
|
+
throw new Error(`W4A16 tensor "${name}" has unsupported scale companion dtype "${scaleCompanion.location?.dtype}".`);
|
|
274
|
+
}
|
|
275
|
+
const expectedScaleBytes = rows * groupsPerRow * scaleBytesPerElement;
|
|
276
|
+
if (scaleCompanion.bytes.byteLength !== expectedScaleBytes) {
|
|
277
|
+
throw new Error(
|
|
278
|
+
`W4A16 tensor "${name}" scale byte length ${scaleCompanion.bytes.byteLength} does not match expected ${expectedScaleBytes}.`
|
|
279
|
+
);
|
|
280
|
+
}
|
|
281
|
+
return {
|
|
282
|
+
rows,
|
|
283
|
+
cols,
|
|
284
|
+
groupsPerRow,
|
|
285
|
+
scaleDtype: scaleDtype.toLowerCase(),
|
|
286
|
+
scaleBytes: scaleCompanion.bytes,
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function dequantizeW4A16ToF16(shardData, location, name) {
|
|
291
|
+
const scaleCompanion = getStorageCompanion(shardData, location, name, 'scales');
|
|
292
|
+
const shapeCompanion = getStorageCompanion(shardData, location, name, 'shape');
|
|
293
|
+
const [rows, cols] = assertW4A16Shape(
|
|
294
|
+
readW4A16LogicalShape(shapeCompanion, location.shape, name),
|
|
295
|
+
location.shape,
|
|
296
|
+
name
|
|
297
|
+
);
|
|
298
|
+
const groupsPerRow = Math.ceil(cols / 32);
|
|
299
|
+
const expectedPackedBytes = rows * groupsPerRow * 16;
|
|
300
|
+
if (shardData.byteLength !== expectedPackedBytes) {
|
|
301
|
+
throw new Error(
|
|
302
|
+
`W4A16 tensor "${name}" packed byte length ${shardData.byteLength} does not match expected ${expectedPackedBytes}.`
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
const expectedScales = rows * groupsPerRow;
|
|
306
|
+
const scales = readW4A16Scales(scaleCompanion, expectedScales, name);
|
|
307
|
+
const out = new Uint16Array(rows * cols);
|
|
308
|
+
for (let row = 0; row < rows; row += 1) {
|
|
309
|
+
for (let group = 0; group < groupsPerRow; group += 1) {
|
|
310
|
+
const scale = scales[(row * groupsPerRow) + group];
|
|
311
|
+
const packedOffset = ((row * groupsPerRow) + group) * 16;
|
|
312
|
+
for (let lane = 0; lane < 32; lane += 1) {
|
|
313
|
+
const col = (group * 32) + lane;
|
|
314
|
+
if (col >= cols) break;
|
|
315
|
+
const byte = shardData[packedOffset + Math.floor(lane / 2)];
|
|
316
|
+
const quant = readOffsetBinaryInt4(byte, (lane % 2) === 1);
|
|
317
|
+
out[(row * cols) + col] = float32ToFloat16(quant * scale);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
|
|
322
|
+
}
|
|
323
|
+
|
|
111
324
|
|
|
112
325
|
export function isPackedQ4K(location) {
|
|
113
|
-
if (!Array.isArray(location.shape) || location.shape.length
|
|
326
|
+
if (!Array.isArray(location.shape) || location.shape.length < 2) {
|
|
114
327
|
return false;
|
|
115
328
|
}
|
|
116
|
-
const
|
|
329
|
+
const cols = location.shape[location.shape.length - 1];
|
|
330
|
+
const rows = location.shape.slice(0, -1).reduce((a, b) => a * b, 1);
|
|
117
331
|
const expectedRowwise = rows * Math.ceil(cols / QK_K) * Q4K_BLOCK_BYTES;
|
|
118
332
|
return location.size < expectedRowwise;
|
|
119
333
|
}
|
|
@@ -145,6 +359,56 @@ function canUseFusedQ4KStorage(location, config) {
|
|
|
145
359
|
return true;
|
|
146
360
|
}
|
|
147
361
|
|
|
362
|
+
export function isLiteRTAffineInt4FusedEligible(location, config) {
|
|
363
|
+
const caps = config?.gpuCapabilities || getKernelCapabilities();
|
|
364
|
+
if (caps?.hasF16 !== true) return false;
|
|
365
|
+
|
|
366
|
+
if (!Array.isArray(location?.shape) || location.shape.length !== 2) return false;
|
|
367
|
+
if (!location?.role) return false;
|
|
368
|
+
if (isEmbeddingRole(location)) return false;
|
|
369
|
+
if (location.role !== 'matmul' && location.role !== 'lm_head') return false;
|
|
370
|
+
if (!shouldDequantizeToF16(location)) return false;
|
|
371
|
+
|
|
372
|
+
const transform = location.sourceTransform;
|
|
373
|
+
if (!transform || typeof transform !== 'object') return false;
|
|
374
|
+
const sourceDtype = String(transform.sourceDtype || '').toUpperCase();
|
|
375
|
+
const targetDtype = String(transform.targetDtype || '').toUpperCase();
|
|
376
|
+
const locationDtype = String(location.dtype || '').toUpperCase();
|
|
377
|
+
const storageEncoding = String(transform.storageEncoding || '').toLowerCase();
|
|
378
|
+
const scale = Number(transform.scale);
|
|
379
|
+
const zeroPoint = Number(transform.zeroPoint);
|
|
380
|
+
const storageEncodingSupported = storageEncoding === 'signed' || storageEncoding === 'offset_binary';
|
|
381
|
+
|
|
382
|
+
return transform.kind === 'affine_dequant'
|
|
383
|
+
&& transform.scheme === 'per_tensor_affine'
|
|
384
|
+
&& sourceDtype === 'INT4'
|
|
385
|
+
&& targetDtype === 'F16'
|
|
386
|
+
&& locationDtype === 'F16'
|
|
387
|
+
&& storageEncodingSupported
|
|
388
|
+
&& Number.isFinite(scale)
|
|
389
|
+
&& Math.abs(scale - 0.0625) <= Number.EPSILON
|
|
390
|
+
&& Number.isSafeInteger(zeroPoint)
|
|
391
|
+
&& zeroPoint === 0;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
export function isW4A16FusedEligible(location, config) {
|
|
395
|
+
const caps = config?.gpuCapabilities || getKernelCapabilities();
|
|
396
|
+
if (caps?.hasF16 !== true) return false;
|
|
397
|
+
|
|
398
|
+
if (!Array.isArray(location?.shape) || location.shape.length !== 2) return false;
|
|
399
|
+
if (String(location?.dtype || '').toUpperCase() !== 'W4A16') return false;
|
|
400
|
+
if (!location?.role) return false;
|
|
401
|
+
if (isEmbeddingRole(location)) return false;
|
|
402
|
+
if (location.role !== 'matmul' && location.role !== 'lm_head') return false;
|
|
403
|
+
if (!shouldDequantizeToF16(location)) return false;
|
|
404
|
+
if (location?.storage?.packing !== 'w4a16') return false;
|
|
405
|
+
if (!Array.isArray(location?.storage?.companions)) return false;
|
|
406
|
+
|
|
407
|
+
const hasScales = location.storage.companions.some((entry) => entry.role === 'scales');
|
|
408
|
+
const hasShape = location.storage.companions.some((entry) => entry.role === 'shape');
|
|
409
|
+
return hasScales && hasShape;
|
|
410
|
+
}
|
|
411
|
+
|
|
148
412
|
function getQ4KDenseMaterializedSizeBytes(location, config) {
|
|
149
413
|
if (!Array.isArray(location.shape) || location.shape.length === 0) {
|
|
150
414
|
return null;
|
|
@@ -293,6 +557,80 @@ export async function loadQ4KFused(shardData, location, name) {
|
|
|
293
557
|
}
|
|
294
558
|
}
|
|
295
559
|
|
|
560
|
+
export async function loadLiteRTInt4Fused(shardData, location, name, config = null) {
|
|
561
|
+
if (isGpuBufferInstance(shardData)) {
|
|
562
|
+
throw new Error(
|
|
563
|
+
`LiteRT INT4 tensor "${name}" requires raw packed source bytes before GPU upload.`
|
|
564
|
+
);
|
|
565
|
+
}
|
|
566
|
+
if (!isLiteRTAffineInt4FusedEligible(location, config ?? { gpuCapabilities: getKernelCapabilities() })) {
|
|
567
|
+
throw new Error(
|
|
568
|
+
`LiteRT INT4 tensor "${name}" does not match the fused fixed-affine contract ` +
|
|
569
|
+
'(INT4 -> F16, storageEncoding=signed|offset_binary, per_tensor_affine, scale=0.0625, zeroPoint=0, 2D matmul/lm_head role).'
|
|
570
|
+
);
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
const [rows, cols] = location.shape;
|
|
574
|
+
const expectedBytes = rows * Math.ceil(cols / 2);
|
|
575
|
+
const actualBytes = resolveInputByteLength(shardData, location.size);
|
|
576
|
+
if (actualBytes !== expectedBytes) {
|
|
577
|
+
throw new Error(
|
|
578
|
+
`LiteRT INT4 tensor "${name}" packed byte size mismatch. ` +
|
|
579
|
+
`Expected ${expectedBytes} bytes for shape [${rows},${cols}], got ${actualBytes}.`
|
|
580
|
+
);
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
const device = getDevice();
|
|
584
|
+
const buffer = acquireAlignedBuffer(actualBytes, `litert_int4_${name}`);
|
|
585
|
+
try {
|
|
586
|
+
writeBufferAligned(device, buffer, shardData);
|
|
587
|
+
return {
|
|
588
|
+
data: createWeightBuffer(buffer, 'litert_int4', 'row', location.shape, name, null, {
|
|
589
|
+
storageEncoding: String(location.sourceTransform.storageEncoding).toLowerCase(),
|
|
590
|
+
}),
|
|
591
|
+
allocatedBuffers: [buffer],
|
|
592
|
+
};
|
|
593
|
+
} catch (error) {
|
|
594
|
+
releaseBuffer(buffer);
|
|
595
|
+
throw error;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
export async function loadW4A16Fused(shardData, location, name, config = null) {
|
|
600
|
+
if (isGpuBufferInstance(shardData)) {
|
|
601
|
+
throw new Error(
|
|
602
|
+
`W4A16 tensor "${name}" requires raw packed source bytes before GPU upload.`
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
if (!isW4A16FusedEligible(location, config ?? { gpuCapabilities: getKernelCapabilities() })) {
|
|
606
|
+
throw new Error(
|
|
607
|
+
`W4A16 tensor "${name}" does not match the fused packed contract ` +
|
|
608
|
+
'(dtype=W4A16, storage.packing=w4a16, scales+shape companions, 2D matmul/lm_head role).'
|
|
609
|
+
);
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
const storage = validateW4A16PackedStorage(shardData, location, name);
|
|
613
|
+
const device = getDevice();
|
|
614
|
+
const weightBuffer = acquireAlignedBuffer(shardData.byteLength, `w4a16_${name}`);
|
|
615
|
+
const scaleBuffer = acquireAlignedBuffer(storage.scaleBytes.byteLength, `w4a16_scales_${name}`);
|
|
616
|
+
try {
|
|
617
|
+
writeBufferAligned(device, weightBuffer, shardData);
|
|
618
|
+
writeBufferAligned(device, scaleBuffer, storage.scaleBytes);
|
|
619
|
+
return {
|
|
620
|
+
data: createWeightBuffer(weightBuffer, 'w4a16', 'row', location.shape, name, null, {
|
|
621
|
+
scaleBuffer,
|
|
622
|
+
scaleDtype: storage.scaleDtype,
|
|
623
|
+
groupsPerRow: storage.groupsPerRow,
|
|
624
|
+
}),
|
|
625
|
+
allocatedBuffers: [weightBuffer, scaleBuffer],
|
|
626
|
+
};
|
|
627
|
+
} catch (error) {
|
|
628
|
+
releaseBuffer(weightBuffer);
|
|
629
|
+
releaseBuffer(scaleBuffer);
|
|
630
|
+
throw error;
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
|
|
296
634
|
async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name, config) {
|
|
297
635
|
let dequantized = null;
|
|
298
636
|
const outputDtype = getQ4KOutputDtype(location, config);
|
|
@@ -341,16 +679,23 @@ async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name,
|
|
|
341
679
|
}
|
|
342
680
|
|
|
343
681
|
let numBlocks = null;
|
|
682
|
+
let rowCount = null;
|
|
683
|
+
let blocksPerRow = null;
|
|
684
|
+
let denseElementCount = null;
|
|
344
685
|
let dequantizedTensor;
|
|
345
686
|
if (needsRowwise) {
|
|
346
|
-
|
|
687
|
+
rowCount = location.shape.slice(0, -1).reduce((a, b) => a * b, 1);
|
|
688
|
+
blocksPerRow = Math.ceil(K / QK_K);
|
|
689
|
+
numBlocks = rowCount * blocksPerRow;
|
|
690
|
+
denseElementCount = rowCount * K;
|
|
347
691
|
debugTrace.loader(
|
|
348
|
-
`Dequantizing ${name} (row-wise): [${
|
|
692
|
+
`Dequantizing ${name} (row-wise): [${rowCount},${K}], K not 256-aligned, ` +
|
|
349
693
|
`outputDtype=${outputDtype}`
|
|
350
694
|
);
|
|
351
|
-
dequantizedTensor = await dequantizeRowwise(quantBuffer,
|
|
695
|
+
dequantizedTensor = await dequantizeRowwise(quantBuffer, rowCount, K, { outputDtype });
|
|
352
696
|
} else {
|
|
353
697
|
numBlocks = Math.ceil(location.size / Q4K_BLOCK_BYTES);
|
|
698
|
+
denseElementCount = numBlocks * QK_K;
|
|
354
699
|
debugTrace.loader(
|
|
355
700
|
`Dequantizing ${name}: size=${location.size}, numBlocks=${numBlocks}, ` +
|
|
356
701
|
`outputDtype=${outputDtype}, expectedOutput=${numBlocks * QK_K * (outputDtype === 'f16' ? 2 : 4)}`
|
|
@@ -369,9 +714,9 @@ async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name,
|
|
|
369
714
|
if (isProbeTarget) {
|
|
370
715
|
try {
|
|
371
716
|
const bytesPerElem = outputDtype === 'f16' ? 2 : 4;
|
|
372
|
-
const requestedOutputBytes =
|
|
717
|
+
const requestedOutputBytes = denseElementCount * bytesPerElem;
|
|
373
718
|
const sampleCount = paritySamples;
|
|
374
|
-
const readSize = Math.min(sampleCount * bytesPerElem, dequantized.size);
|
|
719
|
+
const readSize = Math.min(sampleCount * bytesPerElem, requestedOutputBytes, dequantized.size);
|
|
375
720
|
const gpuRaw = await readBuffer(dequantized, readSize);
|
|
376
721
|
const gpuBytes = gpuRaw instanceof ArrayBuffer
|
|
377
722
|
? new Uint8Array(gpuRaw)
|
|
@@ -407,7 +752,8 @@ async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name,
|
|
|
407
752
|
|
|
408
753
|
log.warn('DequantProbe',
|
|
409
754
|
`tensor="${name}" shape=[${location.shape}] ` +
|
|
410
|
-
`location.size=${location.size} numBlocks=${numBlocks}
|
|
755
|
+
`location.size=${location.size} numBlocks=${numBlocks} blocksPerRow=${blocksPerRow} ` +
|
|
756
|
+
`rows=${rowCount} K=${K} denseElements=${denseElementCount} outputDtype=${outputDtype} ` +
|
|
411
757
|
`bytesPerElem=${bytesPerElem} requestedOutputBytes=${requestedOutputBytes} bufSize=${dequantized.size} ` +
|
|
412
758
|
`runParity=true sampleCount=${sampleCount}`
|
|
413
759
|
);
|
|
@@ -509,10 +855,10 @@ async function loadQ4KMixed(shardData, location, name, config) {
|
|
|
509
855
|
|
|
510
856
|
function getQ4KCpuReferenceContext(shardData, location, config) {
|
|
511
857
|
const outputDtype = getQ4KOutputDtype(location, config);
|
|
512
|
-
const
|
|
513
|
-
const K =
|
|
514
|
-
const needsRowwise = is2DMatrix && K > 0 && K % QK_K !== 0;
|
|
858
|
+
const isMatrixLike = Array.isArray(location.shape) && location.shape.length >= 2;
|
|
859
|
+
const K = isMatrixLike ? location.shape[location.shape.length - 1] : 0;
|
|
515
860
|
const layout = getWeightLayout(location, config);
|
|
861
|
+
const needsRowwise = isMatrixLike && layout === 'row' && K > 0 && K % QK_K !== 0;
|
|
516
862
|
const eligible = outputDtype === 'f32'
|
|
517
863
|
&& !isGpuBufferInstance(shardData)
|
|
518
864
|
&& (!needsRowwise || layout === 'row');
|
|
@@ -728,12 +1074,58 @@ export async function loadFloat(shardData, location, name, config) {
|
|
|
728
1074
|
}
|
|
729
1075
|
}
|
|
730
1076
|
|
|
1077
|
+
export async function loadW4A16Dequant(shardData, location, name, config) {
|
|
1078
|
+
if (!config) {
|
|
1079
|
+
throw new Error('Tensor load config is required.');
|
|
1080
|
+
}
|
|
1081
|
+
if (isGpuBufferInstance(shardData)) {
|
|
1082
|
+
throw new Error(
|
|
1083
|
+
`W4A16 tensor "${name}" requires CPU-side storage companion materialization before GPU upload.`
|
|
1084
|
+
);
|
|
1085
|
+
}
|
|
1086
|
+
const f16Bytes = dequantizeW4A16ToF16(shardData, location, name);
|
|
1087
|
+
const device = getDevice();
|
|
1088
|
+
const buffer = acquireAlignedBuffer(f16Bytes.byteLength, `w4a16_dequant_${name}`);
|
|
1089
|
+
try {
|
|
1090
|
+
writeBufferAligned(device, buffer, f16Bytes);
|
|
1091
|
+
const layout = selectRuleValue('loader', 'weights', 'weightLayout', {
|
|
1092
|
+
layout: location.layout ?? null,
|
|
1093
|
+
useColumnWise: false,
|
|
1094
|
+
});
|
|
1095
|
+
if (shouldDequantizeToF16(location)) {
|
|
1096
|
+
return {
|
|
1097
|
+
data: createWeightBuffer(buffer, 'f16', layout, location.shape, name),
|
|
1098
|
+
allocatedBuffers: [buffer],
|
|
1099
|
+
};
|
|
1100
|
+
}
|
|
1101
|
+
return {
|
|
1102
|
+
data: applyBufferLayout(buffer, location, 'f16'),
|
|
1103
|
+
allocatedBuffers: [buffer],
|
|
1104
|
+
};
|
|
1105
|
+
} catch (error) {
|
|
1106
|
+
releaseBuffer(buffer);
|
|
1107
|
+
throw error;
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
|
|
731
1111
|
// ============================================================================
|
|
732
1112
|
// Main GPU Loading Entry Point
|
|
733
1113
|
// ============================================================================
|
|
734
1114
|
|
|
735
1115
|
|
|
736
1116
|
const GPU_LOADER_DISPATCH = {
|
|
1117
|
+
functional_descriptor: (shardData, location, name, config) => {
|
|
1118
|
+
debugTrace.loader(`Loading functional descriptor: ${name}`);
|
|
1119
|
+
return loadFunctionalDescriptor(shardData, location, name, config);
|
|
1120
|
+
},
|
|
1121
|
+
litert_int4_fused: (shardData, location, name, config) => {
|
|
1122
|
+
debugTrace.loader(`Loading LiteRT INT4 weight (fused): ${name} (size=${location.size})`);
|
|
1123
|
+
return loadLiteRTInt4Fused(shardData, location, name, config);
|
|
1124
|
+
},
|
|
1125
|
+
w4a16_fused: (shardData, location, name, config) => {
|
|
1126
|
+
debugTrace.loader(`Loading W4A16 weight (fused): ${name} (size=${location.size})`);
|
|
1127
|
+
return loadW4A16Fused(shardData, location, name, config);
|
|
1128
|
+
},
|
|
737
1129
|
q4k_mixed: (shardData, location, name, config) => loadQ4KMixed(shardData, location, name, config),
|
|
738
1130
|
q4k_fused: (shardData, location, name, _config) => {
|
|
739
1131
|
debugTrace.loader(`Loading Q4K weight (fused): ${name} (size=${location.size})`);
|
|
@@ -759,8 +1151,15 @@ const GPU_LOADER_DISPATCH = {
|
|
|
759
1151
|
}
|
|
760
1152
|
),
|
|
761
1153
|
q6k: (shardData, location, name, _config) => loadQ6K(shardData, location, name),
|
|
1154
|
+
w4a16_dequant_reference: (shardData, location, name, config) => loadW4A16Dequant(shardData, location, name, config),
|
|
762
1155
|
bf16: (shardData, location, name, config) => loadBF16(shardData, location, name, config),
|
|
763
1156
|
float: (shardData, location, name, config) => loadFloat(shardData, location, name, config),
|
|
1157
|
+
unsupported_packed_quantization: (_shardData, location, name, _config) => {
|
|
1158
|
+
throw new Error(
|
|
1159
|
+
`Unsupported packed quantization dtype "${location.dtype}" for tensor "${name}". ` +
|
|
1160
|
+
'Add a native loader and kernel path before enabling runtime execution.'
|
|
1161
|
+
);
|
|
1162
|
+
},
|
|
764
1163
|
};
|
|
765
1164
|
|
|
766
1165
|
export async function loadTensorToGPU(shardData, location, name, config) {
|
|
@@ -774,9 +1173,13 @@ export async function loadTensorToGPU(shardData, location, name, config) {
|
|
|
774
1173
|
const q4kBasicBackendClass = platformId === 'basic'
|
|
775
1174
|
|| (caps?.hasSubgroups !== true && caps?.hasF16 !== true);
|
|
776
1175
|
const q4kLimitFallback = resolveQ4KLimitFallback(location, config);
|
|
1176
|
+
const litertAffineInt4FusedEligible = isLiteRTAffineInt4FusedEligible(location, { ...config, gpuCapabilities: caps });
|
|
1177
|
+
const w4a16FusedEligible = isW4A16FusedEligible(location, { ...config, gpuCapabilities: caps });
|
|
777
1178
|
const loaderPath = selectRuleValue('loader', 'tensorLoader', 'gpuLoaderPath', {
|
|
778
1179
|
dtype,
|
|
779
1180
|
role: location.role ?? null,
|
|
1181
|
+
litertAffineInt4FusedEligible,
|
|
1182
|
+
w4a16FusedEligible,
|
|
780
1183
|
useFusedQ4K,
|
|
781
1184
|
requiresFusedQ4KRole,
|
|
782
1185
|
q4kMaterializationMode: config.q4kMaterializationMode ?? 'dense',
|
|
@@ -797,24 +1200,37 @@ export async function loadTensorToGPU(shardData, location, name, config) {
|
|
|
797
1200
|
|
|
798
1201
|
|
|
799
1202
|
const CPU_LOADER_DISPATCH = {
|
|
1203
|
+
unsupported_functional_descriptor: (_shardData, _location, name) => {
|
|
1204
|
+
throw new Error(
|
|
1205
|
+
`FUNCTIONAL_DESCRIPTOR tensor "${name ?? 'unknown'}" requires GPU materialization through loadTensorToGPU.`
|
|
1206
|
+
);
|
|
1207
|
+
},
|
|
800
1208
|
raw: (shardData, _location) => shardData,
|
|
801
|
-
|
|
802
|
-
const
|
|
803
|
-
return
|
|
1209
|
+
w4a16_dequant_reference: (shardData, location) => {
|
|
1210
|
+
const f16Bytes = dequantizeW4A16ToF16(shardData, location, 'cpu');
|
|
1211
|
+
return convertF16ToF32CPU(toUint16View(f16Bytes, 'W4A16 CPU dequantized tensor load'));
|
|
804
1212
|
},
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
1213
|
+
unsupported_packed_quantization: (_shardData, location) => {
|
|
1214
|
+
throw new Error(
|
|
1215
|
+
`Unsupported packed quantization dtype "${location.dtype}" for CPU tensor load. ` +
|
|
1216
|
+
'Add a native loader before enabling runtime execution.'
|
|
1217
|
+
);
|
|
808
1218
|
},
|
|
809
|
-
|
|
1219
|
+
bf16_to_f32: (shardData, _location) => convertBF16ToF32CPU(
|
|
1220
|
+
toUint16View(shardData, 'BF16 CPU tensor load')
|
|
1221
|
+
),
|
|
1222
|
+
f16_to_f32: (shardData, _location) => convertF16ToF32CPU(
|
|
1223
|
+
toUint16View(shardData, 'F16 CPU tensor load')
|
|
1224
|
+
),
|
|
1225
|
+
f32: (shardData, _location) => toFloat32View(shardData, 'F32 CPU tensor load'),
|
|
810
1226
|
};
|
|
811
1227
|
|
|
812
|
-
export function loadTensorToCPU(shardData, location) {
|
|
1228
|
+
export function loadTensorToCPU(shardData, location, name = null) {
|
|
813
1229
|
const dtype = location.dtype;
|
|
814
1230
|
const loaderPath = selectRuleValue('loader', 'tensorLoader', 'cpuLoaderPath', { dtype });
|
|
815
1231
|
const loader = CPU_LOADER_DISPATCH[loaderPath];
|
|
816
1232
|
if (!loader) {
|
|
817
1233
|
throw new Error(`Unknown CPU loader path: "${loaderPath}" for dtype "${dtype}"`);
|
|
818
1234
|
}
|
|
819
|
-
return loader(shardData, location);
|
|
1235
|
+
return loader(shardData, location, name);
|
|
820
1236
|
}
|
|
@@ -18,7 +18,10 @@ export declare function assembleShardData(
|
|
|
18
18
|
location: TensorLocation,
|
|
19
19
|
name: string,
|
|
20
20
|
loadShard: (index: number, options?: ShardLoadOptions) => Promise<ArrayBuffer>,
|
|
21
|
-
loadShardRange?: (index: number, offset: number, length: number) => Promise<ArrayBuffer
|
|
21
|
+
loadShardRange?: (index: number, offset: number, length: number) => Promise<ArrayBuffer>,
|
|
22
|
+
options?: {
|
|
23
|
+
materializeSourceTransform?: boolean;
|
|
24
|
+
}
|
|
22
25
|
): Promise<Uint8Array>;
|
|
23
26
|
|
|
24
27
|
export declare function loadTensorRange(
|
|
@@ -205,7 +205,8 @@ async function materializeLocationBytes(rawBytes, location, name, loadShard, loa
|
|
|
205
205
|
return materializeTensorSourceTransform(rawBytes, location, name);
|
|
206
206
|
}
|
|
207
207
|
|
|
208
|
-
export async function assembleShardData(location, name, loadShard, loadShardRange = null) {
|
|
208
|
+
export async function assembleShardData(location, name, loadShard, loadShardRange = null, options = {}) {
|
|
209
|
+
const shouldMaterializeSourceTransform = options.materializeSourceTransform !== false;
|
|
209
210
|
const spans = getLocationSpans(location);
|
|
210
211
|
if (spans) {
|
|
211
212
|
trace.loader(`Assembling tensor "${name}" from ${spans.length} spans`);
|
|
@@ -240,7 +241,9 @@ export async function assembleShardData(location, name, loadShard, loadShardRang
|
|
|
240
241
|
combined.set(chunk, offset);
|
|
241
242
|
offset += chunk.length;
|
|
242
243
|
}
|
|
243
|
-
return
|
|
244
|
+
return shouldMaterializeSourceTransform
|
|
245
|
+
? materializeLocationBytes(combined, location, name, loadShard, loadShardRange)
|
|
246
|
+
: combined;
|
|
244
247
|
}
|
|
245
248
|
|
|
246
249
|
// Single shard - use view to avoid copying
|
|
@@ -255,7 +258,9 @@ export async function assembleShardData(location, name, loadShard, loadShardRang
|
|
|
255
258
|
);
|
|
256
259
|
}
|
|
257
260
|
const bytes = new Uint8Array(slice, 0, size);
|
|
258
|
-
return
|
|
261
|
+
return shouldMaterializeSourceTransform
|
|
262
|
+
? materializeLocationBytes(bytes, location, name, loadShard, loadShardRange)
|
|
263
|
+
: bytes;
|
|
259
264
|
}
|
|
260
265
|
|
|
261
266
|
const fullShard = await loadShard(shardIndex);
|
|
@@ -265,7 +270,9 @@ export async function assembleShardData(location, name, loadShard, loadShardRang
|
|
|
265
270
|
);
|
|
266
271
|
}
|
|
267
272
|
const bytes = new Uint8Array(fullShard, offset, size);
|
|
268
|
-
return
|
|
273
|
+
return shouldMaterializeSourceTransform
|
|
274
|
+
? materializeLocationBytes(bytes, location, name, loadShard, loadShardRange)
|
|
275
|
+
: bytes;
|
|
269
276
|
}
|
|
270
277
|
|
|
271
278
|
export async function loadTensorRange(location, name, byteOffset, byteLength, loadShardRange) {
|
package/src/loader/weights.d.ts
CHANGED
|
@@ -10,9 +10,11 @@ export interface ExpertWeights {
|
|
|
10
10
|
gate?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
11
11
|
up?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
12
12
|
down?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
13
|
-
|
|
13
|
+
gateUp?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
14
|
+
expertFormat?: 'mixtral' | 'gpt-oss' | 'gemma4';
|
|
14
15
|
expertIdx?: number;
|
|
15
16
|
numExperts?: number;
|
|
17
|
+
expertIntermediateSize?: number;
|
|
16
18
|
gateUpBlocks?: GPUBuffer | null;
|
|
17
19
|
gateUpScales?: GPUBuffer | null;
|
|
18
20
|
gateUpBias?: GPUBuffer | null;
|