doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { getDevice } from '../gpu/device.js';
|
|
2
|
+
import { DTYPE_SIZES } from '../config/schema/index.js';
|
|
3
|
+
import { selectRuleValue } from '../rules/rule-registry.js';
|
|
4
|
+
import { getLargeWeightMaxBytes } from './manifest-config.js';
|
|
5
|
+
|
|
6
|
+
export const MAX_SPLIT4_EMBEDDING_SECTIONS = 4;
|
|
7
|
+
export const MAX_SPLIT8_EMBEDDING_SECTIONS = 8;
|
|
8
|
+
export const MAX_SPLIT_EMBEDDING_SECTIONS = MAX_SPLIT8_EMBEDDING_SECTIONS;
|
|
9
|
+
export const SPLIT_EMBEDDING_STORAGE_BUFFER_OVERHEAD = 2;
|
|
10
|
+
|
|
11
|
+
export function getEmbeddingFloatDtype(location) {
|
|
12
|
+
return selectRuleValue('loader', 'weights', 'floatLocationDtype', {
|
|
13
|
+
locationDtype: location?.dtype,
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function expectsSplitGpuEmbeddingKernel(embeddingKernel) {
|
|
18
|
+
return getSplitGpuEmbeddingKernelSectionCount(embeddingKernel) > 0;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function getSplitGpuEmbeddingKernelSectionCount(embeddingKernel) {
|
|
22
|
+
const kernel = embeddingKernel?.kernel;
|
|
23
|
+
const entry = embeddingKernel?.entry;
|
|
24
|
+
if (kernel === 'gather_split4_f16_vec4_f16_out.wgsl' && entry === 'gather_vec4_f16_out') {
|
|
25
|
+
return MAX_SPLIT4_EMBEDDING_SECTIONS;
|
|
26
|
+
}
|
|
27
|
+
if (kernel === 'gather_split8_f16_vec4_f16_out.wgsl' && entry === 'gather_vec4_f16_out') {
|
|
28
|
+
return MAX_SPLIT8_EMBEDDING_SECTIONS;
|
|
29
|
+
}
|
|
30
|
+
if (kernel === 'gather_split8_f16_vec4_f32_out.wgsl' && entry === 'gather_vec4_f32_out') {
|
|
31
|
+
return MAX_SPLIT8_EMBEDDING_SECTIONS;
|
|
32
|
+
}
|
|
33
|
+
return 0;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function getSplitGpuEmbeddingRequiredStorageBuffers(sectionCount) {
|
|
37
|
+
return sectionCount + SPLIT_EMBEDDING_STORAGE_BUFFER_OVERHEAD;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function getMaxSplitGpuEmbeddingSectionsForDevice(embeddingKernel = null, device = getDevice()) {
|
|
41
|
+
const kernelSections = getSplitGpuEmbeddingKernelSectionCount(embeddingKernel);
|
|
42
|
+
const maxStorageBuffersPerShaderStage = device?.limits?.maxStorageBuffersPerShaderStage;
|
|
43
|
+
if (kernelSections > 0) {
|
|
44
|
+
const requiredStorageBuffers = getSplitGpuEmbeddingRequiredStorageBuffers(kernelSections);
|
|
45
|
+
if (
|
|
46
|
+
Number.isFinite(maxStorageBuffersPerShaderStage)
|
|
47
|
+
&& maxStorageBuffersPerShaderStage < requiredStorageBuffers
|
|
48
|
+
) {
|
|
49
|
+
return 0;
|
|
50
|
+
}
|
|
51
|
+
return kernelSections;
|
|
52
|
+
}
|
|
53
|
+
if (!Number.isFinite(maxStorageBuffersPerShaderStage)) {
|
|
54
|
+
return MAX_SPLIT_EMBEDDING_SECTIONS;
|
|
55
|
+
}
|
|
56
|
+
return Math.max(
|
|
57
|
+
0,
|
|
58
|
+
Math.min(
|
|
59
|
+
MAX_SPLIT_EMBEDDING_SECTIONS,
|
|
60
|
+
maxStorageBuffersPerShaderStage - SPLIT_EMBEDDING_STORAGE_BUFFER_OVERHEAD
|
|
61
|
+
)
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function alignByteLength(byteLength) {
|
|
66
|
+
return Math.ceil(byteLength / 4) * 4;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function estimateEmbeddingTensorBytes(location) {
|
|
70
|
+
if (!location?.shape || location.shape.length !== 2) {
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
const [rows, hidden] = location.shape;
|
|
74
|
+
if (!Number.isFinite(rows) || rows <= 0 || !Number.isFinite(hidden) || hidden <= 0) {
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
const dtype = getEmbeddingFloatDtype(location);
|
|
78
|
+
const bytesPerElement = DTYPE_SIZES[dtype];
|
|
79
|
+
if (!Number.isFinite(bytesPerElement) || bytesPerElement <= 0) {
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
const tensorSizeBytes = rows * hidden * bytesPerElement;
|
|
83
|
+
if (!Number.isFinite(tensorSizeBytes) || tensorSizeBytes <= 0) {
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
return {
|
|
87
|
+
dtype,
|
|
88
|
+
rows,
|
|
89
|
+
hidden,
|
|
90
|
+
bytesPerElement,
|
|
91
|
+
tensorSizeBytes,
|
|
92
|
+
alignedTensorSizeBytes: alignByteLength(tensorSizeBytes),
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function createGpuResidentEmbeddingLimitError({ name, location, embeddingKernel = null }) {
|
|
97
|
+
const estimate = estimateEmbeddingTensorBytes(location);
|
|
98
|
+
if (!estimate) {
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
const device = getDevice();
|
|
102
|
+
const maxStorageBufferBindingSize = device?.limits?.maxStorageBufferBindingSize;
|
|
103
|
+
const maxBufferSize = device?.limits?.maxBufferSize;
|
|
104
|
+
const maxStorageBuffersPerShaderStage = device?.limits?.maxStorageBuffersPerShaderStage;
|
|
105
|
+
if (
|
|
106
|
+
!Number.isFinite(maxStorageBufferBindingSize)
|
|
107
|
+
|| maxStorageBufferBindingSize <= 0
|
|
108
|
+
|| !Number.isFinite(maxBufferSize)
|
|
109
|
+
|| maxBufferSize <= 0
|
|
110
|
+
) {
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
const maxGpuResidentBytes = Math.min(maxStorageBufferBindingSize, maxBufferSize);
|
|
114
|
+
if (estimate.alignedTensorSizeBytes <= maxGpuResidentBytes) {
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const rowBytes = estimate.hidden * estimate.bytesPerElement;
|
|
119
|
+
const largeWeightMaxBytes = getLargeWeightMaxBytes();
|
|
120
|
+
const rowsPerSplitSection = largeWeightMaxBytes
|
|
121
|
+
? Math.floor(largeWeightMaxBytes / rowBytes)
|
|
122
|
+
: null;
|
|
123
|
+
const requiredSplitSections = rowsPerSplitSection && rowsPerSplitSection > 0
|
|
124
|
+
? Math.ceil(estimate.rows / rowsPerSplitSection)
|
|
125
|
+
: null;
|
|
126
|
+
const splitKernelExpected = expectsSplitGpuEmbeddingKernel(embeddingKernel);
|
|
127
|
+
const activeSplitKernelMaxSections = getSplitGpuEmbeddingKernelSectionCount(embeddingKernel) || null;
|
|
128
|
+
const maxSplitEmbeddingSections = getMaxSplitGpuEmbeddingSectionsForDevice(embeddingKernel, device);
|
|
129
|
+
if (
|
|
130
|
+
splitKernelExpected
|
|
131
|
+
&& Number.isFinite(requiredSplitSections)
|
|
132
|
+
&& requiredSplitSections <= maxSplitEmbeddingSections
|
|
133
|
+
) {
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
const error = new Error(
|
|
137
|
+
`[Loader] Embedding "${name}" cannot be GPU-resident on this device: ` +
|
|
138
|
+
`requires ${estimate.alignedTensorSizeBytes} bytes but max storage binding is ` +
|
|
139
|
+
`${maxGpuResidentBytes} bytes. Configure a split embedding kernel that fits this device ` +
|
|
140
|
+
'or use a device with a larger storage binding limit.'
|
|
141
|
+
);
|
|
142
|
+
error.details = {
|
|
143
|
+
weightLoadFailure: {
|
|
144
|
+
tensorName: name,
|
|
145
|
+
tensorRole: location.role ?? null,
|
|
146
|
+
tensorDtype: location.dtype ?? null,
|
|
147
|
+
tensorShape: Array.isArray(location.shape) ? [...location.shape] : null,
|
|
148
|
+
tensorSizeBytes: estimate.alignedTensorSizeBytes,
|
|
149
|
+
tensorLoadStage: 'gpuResidentEmbeddingLimitPreflight',
|
|
150
|
+
toGPU: true,
|
|
151
|
+
streamedUpload: false,
|
|
152
|
+
deviceLimitFailure: {
|
|
153
|
+
kind: 'gpu_resident_embedding_exceeds_device_limit',
|
|
154
|
+
maxGpuResidentBytes,
|
|
155
|
+
maxStorageBufferBindingSize,
|
|
156
|
+
maxBufferSize,
|
|
157
|
+
maxStorageBuffersPerShaderStage: Number.isFinite(maxStorageBuffersPerShaderStage)
|
|
158
|
+
? maxStorageBuffersPerShaderStage
|
|
159
|
+
: null,
|
|
160
|
+
largeWeightMaxBytes,
|
|
161
|
+
embeddingKernel: embeddingKernel
|
|
162
|
+
? {
|
|
163
|
+
kernel: embeddingKernel.kernel ?? null,
|
|
164
|
+
entry: embeddingKernel.entry ?? null,
|
|
165
|
+
}
|
|
166
|
+
: null,
|
|
167
|
+
splitKernelExpected,
|
|
168
|
+
activeSplitKernelMaxSections,
|
|
169
|
+
maxSplitEmbeddingSections,
|
|
170
|
+
requiredSplitSections,
|
|
171
|
+
},
|
|
172
|
+
},
|
|
173
|
+
};
|
|
174
|
+
return error;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function resolveLargeWeightGpuResidentOverrides(manifestOverrides, runtimeConfig) {
|
|
178
|
+
const runtimeOverrides = runtimeConfig?.inference?.largeWeights?.gpuResidentOverrides;
|
|
179
|
+
if (Array.isArray(runtimeOverrides)) {
|
|
180
|
+
return runtimeOverrides;
|
|
181
|
+
}
|
|
182
|
+
if (Array.isArray(manifestOverrides)) {
|
|
183
|
+
return manifestOverrides;
|
|
184
|
+
}
|
|
185
|
+
return null;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
export function resolveManifestGpuResidentEmbeddingLimitError(manifest, options = {}) {
|
|
189
|
+
const tensorLocations = options.storageManifest?.tensors ?? manifest?.tensors;
|
|
190
|
+
if (!tensorLocations || typeof tensorLocations !== 'object') {
|
|
191
|
+
return null;
|
|
192
|
+
}
|
|
193
|
+
const overrides = resolveLargeWeightGpuResidentOverrides(
|
|
194
|
+
manifest?.inference?.largeWeights?.gpuResidentOverrides,
|
|
195
|
+
options.runtimeConfig
|
|
196
|
+
);
|
|
197
|
+
if (!Array.isArray(overrides) || overrides.length === 0) {
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
const embeddingKernel = manifest?.inference?.execution?.kernels?.embed ?? null;
|
|
201
|
+
for (const name of overrides) {
|
|
202
|
+
const location = tensorLocations[name];
|
|
203
|
+
if (location?.role !== 'embedding') {
|
|
204
|
+
continue;
|
|
205
|
+
}
|
|
206
|
+
const error = createGpuResidentEmbeddingLimitError({
|
|
207
|
+
name,
|
|
208
|
+
location,
|
|
209
|
+
embeddingKernel,
|
|
210
|
+
});
|
|
211
|
+
if (error) {
|
|
212
|
+
return error;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
return null;
|
|
216
|
+
}
|
|
@@ -14,6 +14,7 @@ import type {
|
|
|
14
14
|
WeightBuffer,
|
|
15
15
|
WeightLayout,
|
|
16
16
|
CpuWeightBuffer,
|
|
17
|
+
SplitWeightBuffer,
|
|
17
18
|
} from '../gpu/weight-buffer.js';
|
|
18
19
|
import type { TensorLocation } from './loader-types.js';
|
|
19
20
|
|
|
@@ -22,7 +23,7 @@ export type TensorLoader = (
|
|
|
22
23
|
name: string,
|
|
23
24
|
toGPU?: boolean,
|
|
24
25
|
silent?: boolean
|
|
25
|
-
) => Promise<GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | Uint8Array | null>;
|
|
26
|
+
) => Promise<GPUBuffer | WeightBuffer | CpuWeightBuffer | SplitWeightBuffer | Float32Array | Uint8Array | null>;
|
|
26
27
|
|
|
27
28
|
/**
|
|
28
29
|
* Context required for embedding loading.
|
|
@@ -46,10 +47,15 @@ export interface EmbeddingLoaderContext {
|
|
|
46
47
|
preserveF32Embeddings?: boolean;
|
|
47
48
|
/** Host shader-f16 capability, used to choose CPU F16->F32 fallback on no-f16 devices */
|
|
48
49
|
hostHasShaderF16?: boolean | null;
|
|
50
|
+
/** Manifest-declared embedding kernel identity. */
|
|
51
|
+
embeddingKernel?: {
|
|
52
|
+
kernel?: string;
|
|
53
|
+
entry?: string;
|
|
54
|
+
} | null;
|
|
49
55
|
}
|
|
50
56
|
|
|
51
57
|
/** Result of embedding loading */
|
|
52
|
-
export type EmbeddingResult = GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
|
|
58
|
+
export type EmbeddingResult = GPUBuffer | WeightBuffer | CpuWeightBuffer | SplitWeightBuffer | Float32Array | null;
|
|
53
59
|
|
|
54
60
|
/**
|
|
55
61
|
* Load embedding weights.
|
|
@@ -3,20 +3,32 @@
|
|
|
3
3
|
import {
|
|
4
4
|
createWeightBuffer,
|
|
5
5
|
createCpuWeightBuffer,
|
|
6
|
+
createSplitWeightBuffer,
|
|
6
7
|
isWeightBuffer,
|
|
7
8
|
isCpuWeightBuffer,
|
|
9
|
+
isSplitWeightBuffer,
|
|
8
10
|
getWeightDtype,
|
|
9
11
|
getLayout,
|
|
10
12
|
} from '../gpu/weight-buffer.js';
|
|
13
|
+
import { getDevice } from '../gpu/device.js';
|
|
11
14
|
import { maybeDowncastToF16 } from './weight-downcast.js';
|
|
12
15
|
import { getTensorNamesByRole } from './tensors/tensor-role.js';
|
|
13
16
|
import { log } from '../debug/index.js';
|
|
14
17
|
import { selectRuleValue } from '../rules/rule-registry.js';
|
|
18
|
+
import { DTYPE_SIZES } from '../config/schema/index.js';
|
|
15
19
|
import { createTensor } from '../gpu/tensor.js';
|
|
16
20
|
import { castF16ToF32 } from '../gpu/kernel-selector.js';
|
|
17
21
|
import { releaseBuffer } from '../memory/buffer-pool.js';
|
|
18
|
-
import { loadTensorRange } from './tensors/tensor-reader.js';
|
|
22
|
+
import { assembleShardData, loadTensorRange } from './tensors/tensor-reader.js';
|
|
19
23
|
import { hasSourceTransform } from './tensors/source-transform.js';
|
|
24
|
+
import { getLargeWeightMaxBytes } from './manifest-config.js';
|
|
25
|
+
import {
|
|
26
|
+
createGpuResidentEmbeddingLimitError,
|
|
27
|
+
expectsSplitGpuEmbeddingKernel,
|
|
28
|
+
getEmbeddingFloatDtype,
|
|
29
|
+
getSplitGpuEmbeddingKernelSectionCount,
|
|
30
|
+
getSplitGpuEmbeddingRequiredStorageBuffers,
|
|
31
|
+
} from './embedding-limit-preflight.js';
|
|
20
32
|
|
|
21
33
|
// ============================================================================
|
|
22
34
|
// Constants
|
|
@@ -25,6 +37,7 @@ import { hasSourceTransform } from './tensors/source-transform.js';
|
|
|
25
37
|
|
|
26
38
|
const EMBEDDING_ROLE = 'embedding';
|
|
27
39
|
const EMBEDDING_GROUP = 'embed';
|
|
40
|
+
const SPLIT_UPLOAD_CHUNK_BYTES = 64 * 1024 * 1024;
|
|
28
41
|
|
|
29
42
|
function isPerLayerEmbeddingTensor(name) {
|
|
30
43
|
return String(name ?? '').toLowerCase().includes('embed_tokens_per_layer.weight');
|
|
@@ -34,6 +47,17 @@ function isGpuBufferInstance(value) {
|
|
|
34
47
|
return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
|
|
35
48
|
}
|
|
36
49
|
|
|
50
|
+
function normalizeLiteRTInt4StorageEncoding(transform, name) {
|
|
51
|
+
const storageEncoding = String(transform?.storageEncoding ?? '').toLowerCase();
|
|
52
|
+
if (storageEncoding !== 'signed' && storageEncoding !== 'offset_binary') {
|
|
53
|
+
throw new Error(
|
|
54
|
+
`[Loader] Embedding "${name}" LiteRT INT4 sourceTransform.storageEncoding must be ` +
|
|
55
|
+
`"signed" or "offset_binary", got "${transform?.storageEncoding ?? 'missing'}".`
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
return storageEncoding;
|
|
59
|
+
}
|
|
60
|
+
|
|
37
61
|
function createRangeBackedTensorSource(ctx, name, location) {
|
|
38
62
|
if (typeof ctx.loadShardRange !== 'function') {
|
|
39
63
|
return null;
|
|
@@ -62,6 +86,219 @@ function createRangeBackedWeightBuffer(ctx, name, location) {
|
|
|
62
86
|
return createCpuWeightBuffer(source, dtype, layout, location.shape, name);
|
|
63
87
|
}
|
|
64
88
|
|
|
89
|
+
function isFixedLiteRTInt4AffineEmbedding(ctx, name, location) {
|
|
90
|
+
if (ctx.hostHasShaderF16 === false) {
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
if (!location?.shape || location.shape.length !== 2) {
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
if (location.role !== EMBEDDING_ROLE) {
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
if (ctx.resolveWeightLayout(location) !== 'row') {
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
const transform = location.sourceTransform;
|
|
103
|
+
if (!transform || transform.kind !== 'affine_dequant') {
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
if (transform.scheme !== 'per_tensor_affine') {
|
|
107
|
+
return false;
|
|
108
|
+
}
|
|
109
|
+
if (String(transform.sourceDtype ?? '').toUpperCase() !== 'INT4') {
|
|
110
|
+
return false;
|
|
111
|
+
}
|
|
112
|
+
if (String(transform.targetDtype ?? '').toUpperCase() !== 'F16') {
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
normalizeLiteRTInt4StorageEncoding(transform, name);
|
|
116
|
+
if (Number(transform.scale) !== 0.0625 || Number(transform.zeroPoint) !== 0) {
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
119
|
+
const [, hiddenSize] = location.shape;
|
|
120
|
+
const expectedBytes = location.shape[0] * Math.ceil(hiddenSize / 2);
|
|
121
|
+
if (location.size !== expectedBytes) {
|
|
122
|
+
throw new Error(
|
|
123
|
+
`[Loader] Embedding "${name}" LiteRT INT4 source size mismatch: ` +
|
|
124
|
+
`manifest size=${location.size}, expected=${expectedBytes}.`
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
return typeof ctx.loadShardRange === 'function';
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
async function createLiteRTInt4GpuEmbeddingWeightBuffer(ctx, name, location) {
|
|
131
|
+
if (!isFixedLiteRTInt4AffineEmbedding(ctx, name, location)) {
|
|
132
|
+
return null;
|
|
133
|
+
}
|
|
134
|
+
const device = getDevice();
|
|
135
|
+
if (!device) {
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
const rawBytes = await assembleShardData(location, name, null, ctx.loadShardRange, {
|
|
139
|
+
materializeSourceTransform: false,
|
|
140
|
+
});
|
|
141
|
+
if (rawBytes.byteLength !== location.size) {
|
|
142
|
+
throw new Error(
|
|
143
|
+
`[Loader] Embedding "${name}" LiteRT INT4 raw read ${rawBytes.byteLength} bytes, ` +
|
|
144
|
+
`expected ${location.size}.`
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
const buffer = device.createBuffer({
|
|
148
|
+
label: `${name}:litert_int4`,
|
|
149
|
+
size: alignByteLength(rawBytes.byteLength),
|
|
150
|
+
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
|
|
151
|
+
});
|
|
152
|
+
try {
|
|
153
|
+
writeBufferInChunks(device.queue, buffer, rawBytes);
|
|
154
|
+
} catch (error) {
|
|
155
|
+
buffer.destroy();
|
|
156
|
+
throw error;
|
|
157
|
+
}
|
|
158
|
+
ctx.gpuBuffers.add(buffer);
|
|
159
|
+
log.info(
|
|
160
|
+
'Loader',
|
|
161
|
+
`Embedding "${name}" stored as packed LiteRT INT4 GPU source ` +
|
|
162
|
+
`(bytes=${rawBytes.byteLength}, layout=row).`
|
|
163
|
+
);
|
|
164
|
+
return createWeightBuffer(buffer, 'litert_int4', 'row', location.shape, name, null, {
|
|
165
|
+
storageEncoding: normalizeLiteRTInt4StorageEncoding(location.sourceTransform, name),
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function alignByteLength(byteLength) {
|
|
170
|
+
return Math.ceil(byteLength / 4) * 4;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function writeBufferInChunks(queue, buffer, bytes) {
|
|
174
|
+
for (let offset = 0; offset < bytes.byteLength; offset += SPLIT_UPLOAD_CHUNK_BYTES) {
|
|
175
|
+
const end = Math.min(offset + SPLIT_UPLOAD_CHUNK_BYTES, bytes.byteLength);
|
|
176
|
+
queue.writeBuffer(buffer, offset, bytes, offset, end - offset);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function expectsSplitGpuEmbedding(ctx) {
|
|
181
|
+
return expectsSplitGpuEmbeddingKernel(ctx.embeddingKernel);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
async function createSplitGpuEmbeddingWeightBuffer(ctx, name, location) {
|
|
185
|
+
if (!expectsSplitGpuEmbedding(ctx)) {
|
|
186
|
+
return null;
|
|
187
|
+
}
|
|
188
|
+
if (ctx.hostHasShaderF16 === false) {
|
|
189
|
+
return null;
|
|
190
|
+
}
|
|
191
|
+
if (!location?.shape || location.shape.length !== 2) {
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
if (hasSourceTransform(location)) {
|
|
195
|
+
return null;
|
|
196
|
+
}
|
|
197
|
+
if (typeof ctx.loadShardRange !== 'function') {
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const layout = ctx.resolveWeightLayout(location);
|
|
202
|
+
if (layout !== 'row') {
|
|
203
|
+
return null;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
const dtype = getEmbeddingFloatDtype(location);
|
|
207
|
+
if (dtype !== 'f16') {
|
|
208
|
+
return null;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const device = getDevice();
|
|
212
|
+
const maxBytes = getLargeWeightMaxBytes();
|
|
213
|
+
if (!device || !maxBytes) {
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
216
|
+
const maxSplitSections = getSplitGpuEmbeddingKernelSectionCount(ctx.embeddingKernel);
|
|
217
|
+
if (maxSplitSections <= 0) {
|
|
218
|
+
return null;
|
|
219
|
+
}
|
|
220
|
+
const requiredStorageBuffers = getSplitGpuEmbeddingRequiredStorageBuffers(maxSplitSections);
|
|
221
|
+
if ((device.limits.maxStorageBuffersPerShaderStage ?? 0) < requiredStorageBuffers) {
|
|
222
|
+
log.warn(
|
|
223
|
+
'Loader',
|
|
224
|
+
`Embedding "${name}" cannot use split GPU residency: ` +
|
|
225
|
+
`maxStorageBuffersPerShaderStage is below ${requiredStorageBuffers}.`
|
|
226
|
+
);
|
|
227
|
+
return null;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
const [rows, hidden] = location.shape;
|
|
231
|
+
const bytesPerElement = DTYPE_SIZES[dtype];
|
|
232
|
+
const rowBytes = hidden * bytesPerElement;
|
|
233
|
+
if (!Number.isFinite(rowBytes) || rowBytes <= 0 || rowBytes > maxBytes) {
|
|
234
|
+
return null;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
const totalBytes = rows * rowBytes;
|
|
238
|
+
if (totalBytes <= maxBytes) {
|
|
239
|
+
return null;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
const rowsPerSection = Math.floor(maxBytes / rowBytes);
|
|
243
|
+
if (rowsPerSection <= 0) {
|
|
244
|
+
return null;
|
|
245
|
+
}
|
|
246
|
+
const sectionCount = Math.ceil(rows / rowsPerSection);
|
|
247
|
+
if (sectionCount > maxSplitSections) {
|
|
248
|
+
log.warn(
|
|
249
|
+
'Loader',
|
|
250
|
+
`Embedding "${name}" needs ${sectionCount} GPU sections; active split kernel supports ` +
|
|
251
|
+
`at most ${maxSplitSections}. ` +
|
|
252
|
+
'Using CPU-backed streaming.'
|
|
253
|
+
);
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
const createdBuffers = [];
|
|
258
|
+
try {
|
|
259
|
+
const sections = [];
|
|
260
|
+
for (let rowStart = 0; rowStart < rows; rowStart += rowsPerSection) {
|
|
261
|
+
const rowCount = Math.min(rowsPerSection, rows - rowStart);
|
|
262
|
+
const byteOffset = rowStart * rowBytes;
|
|
263
|
+
const byteLength = rowCount * rowBytes;
|
|
264
|
+
const bytes = await loadTensorRange(location, name, byteOffset, byteLength, ctx.loadShardRange);
|
|
265
|
+
if (bytes.byteLength !== byteLength) {
|
|
266
|
+
throw new Error(
|
|
267
|
+
`[Loader] Embedding "${name}" split section read ${bytes.byteLength} bytes, expected ${byteLength}.`
|
|
268
|
+
);
|
|
269
|
+
}
|
|
270
|
+
const buffer = device.createBuffer({
|
|
271
|
+
label: `${name}:split:${sections.length}`,
|
|
272
|
+
size: alignByteLength(byteLength),
|
|
273
|
+
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
|
|
274
|
+
});
|
|
275
|
+
createdBuffers.push(buffer);
|
|
276
|
+
writeBufferInChunks(device.queue, buffer, bytes);
|
|
277
|
+
sections.push({ buffer, rowStart, rowCount });
|
|
278
|
+
}
|
|
279
|
+
for (const buffer of createdBuffers) {
|
|
280
|
+
ctx.gpuBuffers.add(buffer);
|
|
281
|
+
}
|
|
282
|
+
log.warn(
|
|
283
|
+
'Loader',
|
|
284
|
+
`Embedding "${name}" stored as split GPU sections (${sections.length} sections, dtype=${dtype}, layout=${layout}).`
|
|
285
|
+
);
|
|
286
|
+
return createSplitWeightBuffer(sections, dtype, layout, location.shape, name, {
|
|
287
|
+
splitGatherSectionCount: maxSplitSections,
|
|
288
|
+
sourceKernel: ctx.embeddingKernel,
|
|
289
|
+
});
|
|
290
|
+
} catch (error) {
|
|
291
|
+
for (const buffer of createdBuffers) {
|
|
292
|
+
try {
|
|
293
|
+
buffer.destroy();
|
|
294
|
+
} catch {
|
|
295
|
+
// Ignore cleanup failure while preserving the original load error.
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
throw error;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
65
302
|
function shouldUseRangeBackedEmbeddingSource(ctx, name, location) {
|
|
66
303
|
if (!location) {
|
|
67
304
|
return false;
|
|
@@ -112,29 +349,42 @@ export async function loadEmbeddings(ctx) {
|
|
|
112
349
|
|
|
113
350
|
for (const name of candidates) {
|
|
114
351
|
const loc = ctx.tensorLocations.get(name);
|
|
115
|
-
const
|
|
352
|
+
const packedLiteRTTensor = await createLiteRTInt4GpuEmbeddingWeightBuffer(ctx, name, loc);
|
|
353
|
+
const shouldStream = packedLiteRTTensor ? false : shouldUseRangeBackedEmbeddingSource(ctx, name, loc);
|
|
354
|
+
const splitGpuTensor = packedLiteRTTensor ? null : await createSplitGpuEmbeddingWeightBuffer(ctx, name, loc);
|
|
355
|
+
if (!packedLiteRTTensor && !splitGpuTensor && !shouldStream) {
|
|
356
|
+
const limitError = createGpuResidentEmbeddingLimitError({
|
|
357
|
+
name,
|
|
358
|
+
location: loc,
|
|
359
|
+
embeddingKernel: ctx.embeddingKernel,
|
|
360
|
+
});
|
|
361
|
+
if (limitError) {
|
|
362
|
+
throw limitError;
|
|
363
|
+
}
|
|
364
|
+
}
|
|
116
365
|
|
|
117
366
|
// Load tensor (to CPU if streaming, to GPU otherwise)
|
|
118
|
-
const tensor = shouldStream
|
|
367
|
+
const tensor = packedLiteRTTensor ?? splitGpuTensor ?? (shouldStream
|
|
119
368
|
? (
|
|
120
369
|
createRangeBackedWeightBuffer(ctx, name, loc)
|
|
121
370
|
?? await ctx.loadTensor(name, false, true)
|
|
122
371
|
)
|
|
123
|
-
: await ctx.loadTensor(name, true, true);
|
|
372
|
+
: await ctx.loadTensor(name, true, true));
|
|
373
|
+
const tensorShouldStream = splitGpuTensor ? false : shouldStream;
|
|
124
374
|
|
|
125
375
|
// Skip if not found
|
|
126
376
|
if (!tensor) continue;
|
|
127
377
|
|
|
128
378
|
// Handle streaming path (CPU)
|
|
129
|
-
if (
|
|
379
|
+
if (tensorShouldStream && !(tensor instanceof Float32Array) && !isCpuWeightBuffer(tensor)) {
|
|
130
380
|
throw new Error(
|
|
131
381
|
`[Loader] Embedding "${name}" too large for GPU and cannot be loaded on CPU (dtype=${loc?.dtype ?? 'unknown'}).`
|
|
132
382
|
);
|
|
133
383
|
}
|
|
134
384
|
|
|
135
385
|
// Handle valid tensor types
|
|
136
|
-
if (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || isCpuWeightBuffer(tensor) || tensor instanceof Float32Array) {
|
|
137
|
-
const result = await processEmbeddingTensor(ctx, tensor, name, loc,
|
|
386
|
+
if (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || isCpuWeightBuffer(tensor) || isSplitWeightBuffer(tensor) || tensor instanceof Float32Array) {
|
|
387
|
+
const result = await processEmbeddingTensor(ctx, tensor, name, loc, tensorShouldStream);
|
|
138
388
|
if (result) {
|
|
139
389
|
return result;
|
|
140
390
|
}
|
|
@@ -163,6 +413,10 @@ async function processEmbeddingTensor(ctx, tensor, name, loc, shouldStream) {
|
|
|
163
413
|
// but loaded with an F32 gather kernel.
|
|
164
414
|
const promoted = await maybePromoteEmbeddingsToF32(ctx, tensor, name, loc);
|
|
165
415
|
|
|
416
|
+
if (isSplitWeightBuffer(promoted)) {
|
|
417
|
+
return promoted;
|
|
418
|
+
}
|
|
419
|
+
|
|
166
420
|
// WeightBuffer already has layout set correctly from _loadTensor
|
|
167
421
|
if (isWeightBuffer(promoted)) {
|
|
168
422
|
return maybeDowncastEmbeddings(ctx, promoted, name, loc);
|
|
@@ -205,6 +459,7 @@ async function maybePromoteEmbeddingsToF32(ctx, current, name, loc) {
|
|
|
205
459
|
if (!ctx.preserveF32Embeddings) return current;
|
|
206
460
|
if (current instanceof Float32Array) return current;
|
|
207
461
|
if (isCpuWeightBuffer(current)) return current;
|
|
462
|
+
if (isSplitWeightBuffer(current)) return current;
|
|
208
463
|
|
|
209
464
|
if (isWeightBuffer(current)) {
|
|
210
465
|
const dtype = getWeightDtype(current);
|
|
@@ -244,7 +499,7 @@ async function maybePromoteEmbeddingsToF32(ctx, current, name, loc) {
|
|
|
244
499
|
|
|
245
500
|
async function maybeDowncastEmbeddings(ctx, current, name, loc) {
|
|
246
501
|
// Can't downcast Float32Array or CpuWeightBuffer
|
|
247
|
-
if (current instanceof Float32Array || isCpuWeightBuffer(current)) {
|
|
502
|
+
if (current instanceof Float32Array || isCpuWeightBuffer(current) || isSplitWeightBuffer(current)) {
|
|
248
503
|
return current;
|
|
249
504
|
}
|
|
250
505
|
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
|
|
11
11
|
import type { RDRRManifest } from '../../formats/rdrr/index.js';
|
|
12
12
|
import type { ExpertCache } from './expert-cache.js';
|
|
13
|
+
import type { TensorLocation } from '../loader-types.js';
|
|
13
14
|
import type { ExpertWeights } from '../weights.js';
|
|
14
15
|
import type { WeightBuffer } from '../../gpu/weight-buffer.js';
|
|
15
16
|
import type { ShardLoadOptions } from '../loader-types.js';
|
|
@@ -38,6 +39,8 @@ export interface ShardCacheInterface {
|
|
|
38
39
|
export interface ExpertLoaderContext {
|
|
39
40
|
/** Model manifest */
|
|
40
41
|
manifest: RDRRManifest | null;
|
|
42
|
+
/** Resolved tensor locations from manifest.tensors or tensors.json */
|
|
43
|
+
tensorLocations?: Map<string, TensorLocation>;
|
|
41
44
|
/** Load a tensor by name */
|
|
42
45
|
loadTensor: TensorLoader;
|
|
43
46
|
/** Load a shard by index */
|
|
@@ -46,7 +49,7 @@ export interface ExpertLoaderContext {
|
|
|
46
49
|
shardCache: ShardCacheInterface;
|
|
47
50
|
/** Expert LRU cache */
|
|
48
51
|
expertCache: ExpertCache | null;
|
|
49
|
-
/** Simple map for
|
|
52
|
+
/** Simple map for packed experts */
|
|
50
53
|
experts: Map<string, ExpertWeights>;
|
|
51
54
|
/** GPU buffers to track for cleanup */
|
|
52
55
|
gpuBuffers: Set<GPUBuffer>;
|