doppler-gpu 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -11,7 +11,13 @@ import {
|
|
|
11
11
|
} from '../storage/shard-manager.js';
|
|
12
12
|
import { clearManifest, parseManifest, setManifest as setCurrentManifest } from '../formats/rdrr/index.js';
|
|
13
13
|
import { initDevice, getDevice, getKernelCapabilities } from '../gpu/device.js';
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
PersistentBufferSet,
|
|
16
|
+
acquireBuffer,
|
|
17
|
+
isBufferActive,
|
|
18
|
+
releaseBuffer,
|
|
19
|
+
forceBufferPoolReclaim,
|
|
20
|
+
} from '../memory/buffer-pool.js';
|
|
15
21
|
import { getExpertCache } from './experts/expert-cache.js';
|
|
16
22
|
import { formatBytes } from '../storage/quota.js';
|
|
17
23
|
import { log, trace as debugTrace } from '../debug/index.js';
|
|
@@ -35,6 +41,7 @@ import {
|
|
|
35
41
|
loadTensorToCPU,
|
|
36
42
|
} from './tensors/tensor-loader.js';
|
|
37
43
|
import { loadEmbeddings } from './embedding-loader.js';
|
|
44
|
+
import { loadPerLayerInputWeights } from './per-layer-input-loader.js';
|
|
38
45
|
import { loadLayer } from './layer-loader.js';
|
|
39
46
|
import { loadFinalWeights } from './final-weights-loader.js';
|
|
40
47
|
import {
|
|
@@ -42,8 +49,15 @@ import {
|
|
|
42
49
|
prefetchExperts as prefetchExpertsFromModule,
|
|
43
50
|
predictNextLayerExperts as predictNextLayerExpertsFromModule,
|
|
44
51
|
} from './experts/expert-loader.js';
|
|
45
|
-
import { loadLoRAWeights as loadLoRAWeightsFromModule } from '../adapters/lora-loader.js';
|
|
46
52
|
import { assembleShardData } from './tensors/tensor-reader.js';
|
|
53
|
+
import { hasSourceTransform } from './tensors/source-transform.js';
|
|
54
|
+
|
|
55
|
+
let loraModulePromise = null;
|
|
56
|
+
|
|
57
|
+
async function getExperimentalLoRAModule() {
|
|
58
|
+
loraModulePromise ??= import('../experimental/adapters/lora-loader.js');
|
|
59
|
+
return loraModulePromise;
|
|
60
|
+
}
|
|
47
61
|
|
|
48
62
|
function hasExpertGroups(manifest) {
|
|
49
63
|
if (!manifest?.groups) return false;
|
|
@@ -113,11 +127,13 @@ export class DopplerLoader {
|
|
|
113
127
|
|
|
114
128
|
embeddingPostprocessor = null;
|
|
115
129
|
|
|
130
|
+
perLayerInputWeights = null;
|
|
131
|
+
|
|
116
132
|
// Memory management
|
|
117
133
|
|
|
118
134
|
heapManager = null;
|
|
119
135
|
|
|
120
|
-
gpuBuffers = new
|
|
136
|
+
gpuBuffers = new PersistentBufferSet();
|
|
121
137
|
|
|
122
138
|
// Expert cache for MoE models (LRU eviction)
|
|
123
139
|
|
|
@@ -138,6 +154,8 @@ export class DopplerLoader {
|
|
|
138
154
|
#loadingConfig;
|
|
139
155
|
#loaderDebug = null;
|
|
140
156
|
|
|
157
|
+
#perLayerInputSession = null;
|
|
158
|
+
|
|
141
159
|
// Fused Q4_K matmul: skip dequantization for matmul weights, use fused kernel
|
|
142
160
|
|
|
143
161
|
useFusedQ4K = false;
|
|
@@ -148,6 +166,8 @@ export class DopplerLoader {
|
|
|
148
166
|
|
|
149
167
|
keepF32Weights = false;
|
|
150
168
|
|
|
169
|
+
q4kMaterializationMode = 'dense';
|
|
170
|
+
|
|
151
171
|
// Internal tracking
|
|
152
172
|
|
|
153
173
|
#normOffsetLogged = false;
|
|
@@ -193,11 +213,16 @@ export class DopplerLoader {
|
|
|
193
213
|
this.#loaderDebug = loaderDebug ?? null;
|
|
194
214
|
}
|
|
195
215
|
|
|
216
|
+
setPerLayerInputSession(sessionConfig) {
|
|
217
|
+
this.#perLayerInputSession = sessionConfig ?? null;
|
|
218
|
+
}
|
|
219
|
+
|
|
196
220
|
|
|
197
221
|
setQ4KConfig(config) {
|
|
198
222
|
this.useFusedQ4K = config.useFusedQ4K;
|
|
199
223
|
this.q4kLayout = config.q4kLayout;
|
|
200
224
|
this.keepF32Weights = config.keepF32Weights;
|
|
225
|
+
this.q4kMaterializationMode = config.q4kMaterializationMode ?? 'dense';
|
|
201
226
|
}
|
|
202
227
|
|
|
203
228
|
|
|
@@ -216,6 +241,29 @@ export class DopplerLoader {
|
|
|
216
241
|
this.#memoryMonitor.start(() => this.#getMemoryState());
|
|
217
242
|
}
|
|
218
243
|
|
|
244
|
+
#assertResidentBudget(phase) {
|
|
245
|
+
const budgetConfig = this.#loadingConfig?.memoryManagement?.budget;
|
|
246
|
+
if (!budgetConfig || budgetConfig.enabled !== true) {
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
const maxResidentBytes = Number(budgetConfig.maxResidentBytes);
|
|
250
|
+
if (!Number.isFinite(maxResidentBytes) || maxResidentBytes <= 0) {
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
if (typeof process === 'undefined' || typeof process.memoryUsage !== 'function') {
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
const rssBytes = process.memoryUsage().rss;
|
|
257
|
+
if (rssBytes <= maxResidentBytes) {
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
throw new Error(
|
|
261
|
+
`Loader resident memory budget exceeded during ${phase}: ` +
|
|
262
|
+
`rss=${formatBytes(rssBytes)}, max=${formatBytes(maxResidentBytes)}. ` +
|
|
263
|
+
'Lower the model working set or raise runtime.loading.memoryManagement.budget.maxResidentBytes.'
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
|
|
219
267
|
|
|
220
268
|
#stopMemoryLogging(phase = 'complete') {
|
|
221
269
|
if (this.#memoryMonitor) {
|
|
@@ -296,12 +344,13 @@ export class DopplerLoader {
|
|
|
296
344
|
const prevLayerShardMap = new Map(this.#layerShardMap);
|
|
297
345
|
|
|
298
346
|
try {
|
|
347
|
+
const { loadLoRAWeights } = await getExperimentalLoRAModule();
|
|
299
348
|
this.manifest = manifest;
|
|
300
349
|
// We must rebuild locations so _loadTensor finds them
|
|
301
350
|
await this.#buildTensorLocations();
|
|
302
351
|
this.#logWeightBreakdown();
|
|
303
352
|
|
|
304
|
-
return await
|
|
353
|
+
return await loadLoRAWeights(
|
|
305
354
|
manifest,
|
|
306
355
|
(name, toGPU, silent) => this.#loadTensor(name, toGPU, silent)
|
|
307
356
|
);
|
|
@@ -363,6 +412,7 @@ export class DopplerLoader {
|
|
|
363
412
|
this.modelId = modelId;
|
|
364
413
|
|
|
365
414
|
this.#startMemoryLogging();
|
|
415
|
+
this.#assertResidentBudget('load start');
|
|
366
416
|
|
|
367
417
|
if (!this.shardCache.hasCustomLoader) {
|
|
368
418
|
await openModelStore(modelId);
|
|
@@ -492,6 +542,7 @@ export class DopplerLoader {
|
|
|
492
542
|
try {
|
|
493
543
|
reportProgress('shards', 0.1, 'Loading embeddings...');
|
|
494
544
|
await this.#loadEmbeddings(onProgress);
|
|
545
|
+
this.#assertResidentBudget('embeddings');
|
|
495
546
|
|
|
496
547
|
const resolveNumLayers = (value) => {
|
|
497
548
|
const normalized = Number(value);
|
|
@@ -565,6 +616,7 @@ export class DopplerLoader {
|
|
|
565
616
|
message: `Layer ${l + 1}/${numLayers}`,
|
|
566
617
|
});
|
|
567
618
|
}
|
|
619
|
+
this.#assertResidentBudget(`layer ${l + 1}`);
|
|
568
620
|
}
|
|
569
621
|
|
|
570
622
|
const layersTotalTime = ((performance.now() - layersStartTime) / 1000).toFixed(2);
|
|
@@ -572,6 +624,7 @@ export class DopplerLoader {
|
|
|
572
624
|
|
|
573
625
|
reportProgress('gpu_transfer', 0.85, 'Loading final weights...');
|
|
574
626
|
await this.#loadFinalWeights(onProgress);
|
|
627
|
+
this.#assertResidentBudget('final weights');
|
|
575
628
|
|
|
576
629
|
if (onProgress) {
|
|
577
630
|
onProgress({ stage: 'complete', progress: 1.0 });
|
|
@@ -674,6 +727,10 @@ export class DopplerLoader {
|
|
|
674
727
|
#prefetchLayerShards(layerIdx) {
|
|
675
728
|
const prefetch = this.#loadingConfig.prefetch;
|
|
676
729
|
if (!prefetch?.enabled) return;
|
|
730
|
+
// Range-capable custom loaders are expected to serve fine-grained tensor reads.
|
|
731
|
+
// Whole-shard prefetch defeats that contract and can force invalid >4 GiB reads
|
|
732
|
+
// for direct-source SafeTensors bundles.
|
|
733
|
+
if (this.shardCache.hasCustomLoader && this.shardCache.canStreamRanges) return;
|
|
677
734
|
|
|
678
735
|
const layersAhead = prefetch.layersAhead;
|
|
679
736
|
if (!Number.isFinite(layersAhead) || layersAhead <= 0) return;
|
|
@@ -734,6 +791,7 @@ export class DopplerLoader {
|
|
|
734
791
|
}
|
|
735
792
|
const config = {
|
|
736
793
|
useFusedQ4K: this.useFusedQ4K,
|
|
794
|
+
q4kMaterializationMode: this.q4kMaterializationMode,
|
|
737
795
|
keepF32Weights: this.keepF32Weights,
|
|
738
796
|
q4kLayout: this.q4kLayout,
|
|
739
797
|
loaderDebug: this.#loaderDebug,
|
|
@@ -767,6 +825,7 @@ export class DopplerLoader {
|
|
|
767
825
|
|
|
768
826
|
#shouldStreamUploadToGPU(location) {
|
|
769
827
|
if (!location?.size || location.size <= 0) return false;
|
|
828
|
+
if (hasSourceTransform(location)) return false;
|
|
770
829
|
if (this.shardCache.hasCustomLoader && !this.shardCache.canStreamRanges) return false;
|
|
771
830
|
const chunkBytes = this.#loadingConfig?.storage?.backend?.streaming?.readChunkBytes ?? 0;
|
|
772
831
|
if (!Number.isFinite(chunkBytes) || chunkBytes <= 0) return false;
|
|
@@ -797,10 +856,27 @@ export class DopplerLoader {
|
|
|
797
856
|
|
|
798
857
|
try {
|
|
799
858
|
let dstOffset = 0;
|
|
800
|
-
|
|
859
|
+
let pendingBytes = null;
|
|
860
|
+
const writeAlignedChunk = (bytes) => {
|
|
861
|
+
if (bytes.byteLength === 0) return;
|
|
801
862
|
device.queue.writeBuffer(raw, dstOffset, bytes, bytes.byteOffset, bytes.byteLength);
|
|
802
863
|
dstOffset += bytes.byteLength;
|
|
803
864
|
};
|
|
865
|
+
const uploadChunk = (bytes) => {
|
|
866
|
+
let merged = bytes;
|
|
867
|
+
if (pendingBytes && pendingBytes.byteLength > 0) {
|
|
868
|
+
merged = new Uint8Array(pendingBytes.byteLength + bytes.byteLength);
|
|
869
|
+
merged.set(pendingBytes, 0);
|
|
870
|
+
merged.set(bytes, pendingBytes.byteLength);
|
|
871
|
+
pendingBytes = null;
|
|
872
|
+
}
|
|
873
|
+
const alignedLength = merged.byteLength - (merged.byteLength % 4);
|
|
874
|
+
if (alignedLength > 0) {
|
|
875
|
+
writeAlignedChunk(merged.subarray(0, alignedLength));
|
|
876
|
+
}
|
|
877
|
+
const remainder = merged.byteLength - alignedLength;
|
|
878
|
+
pendingBytes = remainder > 0 ? merged.slice(alignedLength) : null;
|
|
879
|
+
};
|
|
804
880
|
const streamRange = (idx, offset, length) => this.shardCache.streamRange(idx, offset, length, { chunkBytes });
|
|
805
881
|
|
|
806
882
|
if (location.spans) {
|
|
@@ -815,6 +891,14 @@ export class DopplerLoader {
|
|
|
815
891
|
}
|
|
816
892
|
}
|
|
817
893
|
|
|
894
|
+
if (pendingBytes && pendingBytes.byteLength > 0) {
|
|
895
|
+
const padded = new Uint8Array(4);
|
|
896
|
+
padded.set(pendingBytes, 0);
|
|
897
|
+
writeAlignedChunk(padded);
|
|
898
|
+
dstOffset -= (4 - pendingBytes.byteLength);
|
|
899
|
+
pendingBytes = null;
|
|
900
|
+
}
|
|
901
|
+
|
|
818
902
|
if (dstOffset !== location.size) {
|
|
819
903
|
throw new Error(
|
|
820
904
|
`Stream upload short read for "${name}": got=${dstOffset}, expected=${location.size}.`
|
|
@@ -844,6 +928,7 @@ export class DopplerLoader {
|
|
|
844
928
|
const ctx = {
|
|
845
929
|
tensorLocations: this.tensorLocations,
|
|
846
930
|
loadTensor: (name, toGPU, silent) => this.#loadTensor(name, toGPU, silent),
|
|
931
|
+
loadShardRange: (index, offset, length) => this.shardCache.loadRange(index, offset, length),
|
|
847
932
|
shouldStreamLargeWeight: (name, loc, label) => this.#shouldStreamLargeWeight(name, loc, label),
|
|
848
933
|
resolveWeightLayout: (loc) => this.#resolveWeightLayout(loc),
|
|
849
934
|
gpuBuffers: this.gpuBuffers,
|
|
@@ -851,9 +936,20 @@ export class DopplerLoader {
|
|
|
851
936
|
// Keep embedding weights in F32 when manifest quantization requires it.
|
|
852
937
|
// gather.wgsl reads embeddings as f32; downcasting here corrupts reads.
|
|
853
938
|
preserveF32Embeddings: String(this.manifest?.quantizationInfo?.embeddings ?? '').toLowerCase() === 'f32',
|
|
939
|
+
hostHasShaderF16: this.gpuCapabilities?.hasF16 ?? null,
|
|
854
940
|
};
|
|
855
941
|
|
|
856
942
|
this.embeddings = await loadEmbeddings(ctx);
|
|
943
|
+
this.perLayerInputWeights = await loadPerLayerInputWeights({
|
|
944
|
+
modelId: this.manifest?.modelId ?? null,
|
|
945
|
+
tensorLocations: this.tensorLocations,
|
|
946
|
+
gpuBuffers: this.gpuBuffers,
|
|
947
|
+
loadTensor: (name, toGPU, silent) => this.#loadTensor(name, toGPU, silent),
|
|
948
|
+
shouldStreamLargeWeight: (name, loc, label) => this.#shouldStreamLargeWeight(name, loc, label),
|
|
949
|
+
loadShardRange: (index, offset, length) => this.shardCache.loadRange(index, offset, length),
|
|
950
|
+
resolveWeightLayout: (loc) => this.#resolveWeightLayout(loc),
|
|
951
|
+
perLayerInputSession: this.#perLayerInputSession,
|
|
952
|
+
}, this.manifest?.architecture ?? null);
|
|
857
953
|
}
|
|
858
954
|
|
|
859
955
|
|
|
@@ -939,6 +1035,7 @@ export class DopplerLoader {
|
|
|
939
1035
|
const ctx = {
|
|
940
1036
|
tensorLocations: this.tensorLocations,
|
|
941
1037
|
loadTensor: (name, toGPU, silent) => this.#loadTensor(name, toGPU, silent),
|
|
1038
|
+
loadShardRange: (index, offset, length) => this.shardCache.loadRange(index, offset, length),
|
|
942
1039
|
needsNormWeightOffset: () => this.#needsNormWeightOffset(),
|
|
943
1040
|
shouldStreamLargeWeight: (name, loc, label) => this.#shouldStreamLargeWeight(name, loc, label),
|
|
944
1041
|
resolveWeightLayout: (loc) => this.#resolveWeightLayout(loc),
|
|
@@ -1055,6 +1152,7 @@ export class DopplerLoader {
|
|
|
1055
1152
|
this.lmHead = null;
|
|
1056
1153
|
this.finalNorm = null;
|
|
1057
1154
|
this.embeddingPostprocessor = null;
|
|
1155
|
+
this.perLayerInputWeights = null;
|
|
1058
1156
|
this.manifest = null;
|
|
1059
1157
|
clearManifest();
|
|
1060
1158
|
this.modelId = null;
|
|
@@ -22,7 +22,7 @@ export type TensorLoader = (
|
|
|
22
22
|
name: string,
|
|
23
23
|
toGPU?: boolean,
|
|
24
24
|
silent?: boolean
|
|
25
|
-
) => Promise<GPUBuffer | WeightBuffer | Float32Array | Uint8Array | null>;
|
|
25
|
+
) => Promise<GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | Uint8Array | null>;
|
|
26
26
|
|
|
27
27
|
/**
|
|
28
28
|
* Context required for embedding loading.
|
|
@@ -34,6 +34,8 @@ export interface EmbeddingLoaderContext {
|
|
|
34
34
|
loadTensor: TensorLoader;
|
|
35
35
|
/** Check if large weight should stream to CPU */
|
|
36
36
|
shouldStreamLargeWeight: (name: string, loc: TensorLocation, label: string) => boolean;
|
|
37
|
+
/** Load a shard byte range for range-backed CPU sources */
|
|
38
|
+
loadShardRange?: (index: number, offset: number, length: number) => Promise<ArrayBuffer>;
|
|
37
39
|
/** Resolve weight layout from location */
|
|
38
40
|
resolveWeightLayout: (loc: TensorLocation) => WeightLayout;
|
|
39
41
|
/** GPU buffers to track for cleanup */
|
|
@@ -42,6 +44,8 @@ export interface EmbeddingLoaderContext {
|
|
|
42
44
|
keepF32Weights: boolean;
|
|
43
45
|
/** Preserve F32 embeddings when manifest quantization requires F32 embedding weights */
|
|
44
46
|
preserveF32Embeddings?: boolean;
|
|
47
|
+
/** Host shader-f16 capability, used to choose CPU F16->F32 fallback on no-f16 devices */
|
|
48
|
+
hostHasShaderF16?: boolean | null;
|
|
45
49
|
}
|
|
46
50
|
|
|
47
51
|
/** Result of embedding loading */
|
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
createWeightBuffer,
|
|
5
5
|
createCpuWeightBuffer,
|
|
6
6
|
isWeightBuffer,
|
|
7
|
+
isCpuWeightBuffer,
|
|
7
8
|
getWeightDtype,
|
|
8
9
|
getLayout,
|
|
9
10
|
} from '../gpu/weight-buffer.js';
|
|
@@ -14,6 +15,7 @@ import { selectRuleValue } from '../rules/rule-registry.js';
|
|
|
14
15
|
import { createTensor } from '../gpu/tensor.js';
|
|
15
16
|
import { castF16ToF32 } from '../gpu/kernel-selector.js';
|
|
16
17
|
import { releaseBuffer } from '../memory/buffer-pool.js';
|
|
18
|
+
import { loadTensorRange } from './tensors/tensor-reader.js';
|
|
17
19
|
|
|
18
20
|
// ============================================================================
|
|
19
21
|
// Constants
|
|
@@ -23,10 +25,42 @@ import { releaseBuffer } from '../memory/buffer-pool.js';
|
|
|
23
25
|
const EMBEDDING_ROLE = 'embedding';
|
|
24
26
|
const EMBEDDING_GROUP = 'embed';
|
|
25
27
|
|
|
28
|
+
function isPerLayerEmbeddingTensor(name) {
|
|
29
|
+
return String(name ?? '').toLowerCase().includes('embed_tokens_per_layer.weight');
|
|
30
|
+
}
|
|
31
|
+
|
|
26
32
|
function isGpuBufferInstance(value) {
|
|
27
33
|
return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
|
|
28
34
|
}
|
|
29
35
|
|
|
36
|
+
function createRangeBackedTensorSource(ctx, name, location) {
|
|
37
|
+
if (typeof ctx.loadShardRange !== 'function') {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
const normalizedLocationDtype = typeof location?.dtype === 'string'
|
|
41
|
+
? location.dtype.toLowerCase()
|
|
42
|
+
: 'f32';
|
|
43
|
+
return {
|
|
44
|
+
kind: 'tensor_range_source',
|
|
45
|
+
sourceDtype: normalizedLocationDtype,
|
|
46
|
+
async loadRange(byteOffset, byteLength) {
|
|
47
|
+
return loadTensorRange(location, name, byteOffset, byteLength, ctx.loadShardRange);
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function createRangeBackedWeightBuffer(ctx, name, location) {
|
|
53
|
+
const source = createRangeBackedTensorSource(ctx, name, location);
|
|
54
|
+
if (!source || !location?.shape || location.shape.length !== 2) {
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
const layout = ctx.resolveWeightLayout(location);
|
|
58
|
+
const dtype = selectRuleValue('loader', 'weights', 'floatLocationDtype', {
|
|
59
|
+
locationDtype: location.dtype,
|
|
60
|
+
});
|
|
61
|
+
return createCpuWeightBuffer(source, dtype, layout, location.shape, name);
|
|
62
|
+
}
|
|
63
|
+
|
|
30
64
|
// ============================================================================
|
|
31
65
|
// Main Function
|
|
32
66
|
// ============================================================================
|
|
@@ -34,9 +68,11 @@ function isGpuBufferInstance(value) {
|
|
|
34
68
|
|
|
35
69
|
export async function loadEmbeddings(ctx) {
|
|
36
70
|
const embeddingNames = getTensorNamesByRole(ctx.tensorLocations, EMBEDDING_ROLE, EMBEDDING_GROUP);
|
|
37
|
-
const
|
|
71
|
+
const groupedCandidates = embeddingNames.length > 0
|
|
38
72
|
? embeddingNames
|
|
39
73
|
: getTensorNamesByRole(ctx.tensorLocations, EMBEDDING_ROLE);
|
|
74
|
+
const filteredCandidates = groupedCandidates.filter((name) => !isPerLayerEmbeddingTensor(name));
|
|
75
|
+
const candidates = filteredCandidates.length > 0 ? filteredCandidates : groupedCandidates;
|
|
40
76
|
|
|
41
77
|
if (candidates.length === 0) {
|
|
42
78
|
throw new Error(
|
|
@@ -47,23 +83,32 @@ export async function loadEmbeddings(ctx) {
|
|
|
47
83
|
|
|
48
84
|
for (const name of candidates) {
|
|
49
85
|
const loc = ctx.tensorLocations.get(name);
|
|
50
|
-
const
|
|
86
|
+
const requiresCpuFloatFallback = loc
|
|
87
|
+
&& String(loc.dtype ?? '').toUpperCase() === 'F16'
|
|
88
|
+
&& ctx.hostHasShaderF16 === false;
|
|
89
|
+
const shouldStream = requiresCpuFloatFallback
|
|
90
|
+
|| (loc ? ctx.shouldStreamLargeWeight(name, loc, 'Embedding') : false);
|
|
51
91
|
|
|
52
92
|
// Load tensor (to CPU if streaming, to GPU otherwise)
|
|
53
|
-
const tensor =
|
|
93
|
+
const tensor = shouldStream
|
|
94
|
+
? (
|
|
95
|
+
createRangeBackedWeightBuffer(ctx, name, loc)
|
|
96
|
+
?? await ctx.loadTensor(name, false, true)
|
|
97
|
+
)
|
|
98
|
+
: await ctx.loadTensor(name, true, true);
|
|
54
99
|
|
|
55
100
|
// Skip if not found
|
|
56
101
|
if (!tensor) continue;
|
|
57
102
|
|
|
58
103
|
// Handle streaming path (CPU)
|
|
59
|
-
if (shouldStream && !(tensor instanceof Float32Array)) {
|
|
104
|
+
if (shouldStream && !(tensor instanceof Float32Array) && !isCpuWeightBuffer(tensor)) {
|
|
60
105
|
throw new Error(
|
|
61
106
|
`[Loader] Embedding "${name}" too large for GPU and cannot be loaded on CPU (dtype=${loc?.dtype ?? 'unknown'}).`
|
|
62
107
|
);
|
|
63
108
|
}
|
|
64
109
|
|
|
65
110
|
// Handle valid tensor types
|
|
66
|
-
if (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || tensor instanceof Float32Array) {
|
|
111
|
+
if (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || isCpuWeightBuffer(tensor) || tensor instanceof Float32Array) {
|
|
67
112
|
const result = await processEmbeddingTensor(ctx, tensor, name, loc, shouldStream);
|
|
68
113
|
if (result) {
|
|
69
114
|
return result;
|
|
@@ -98,6 +143,11 @@ async function processEmbeddingTensor(ctx, tensor, name, loc, shouldStream) {
|
|
|
98
143
|
return maybeDowncastEmbeddings(ctx, promoted, name, loc);
|
|
99
144
|
}
|
|
100
145
|
|
|
146
|
+
if (isCpuWeightBuffer(promoted)) {
|
|
147
|
+
log.warn('Loader', `Embeddings stored on CPU via range-backed source (layout=${promoted.layout})`);
|
|
148
|
+
return promoted;
|
|
149
|
+
}
|
|
150
|
+
|
|
101
151
|
// Float32Array streaming path
|
|
102
152
|
if (promoted instanceof Float32Array && loc?.shape && shouldStream) {
|
|
103
153
|
const layout = ctx.resolveWeightLayout(loc);
|
|
@@ -129,6 +179,7 @@ async function processEmbeddingTensor(ctx, tensor, name, loc, shouldStream) {
|
|
|
129
179
|
async function maybePromoteEmbeddingsToF32(ctx, current, name, loc) {
|
|
130
180
|
if (!ctx.preserveF32Embeddings) return current;
|
|
131
181
|
if (current instanceof Float32Array) return current;
|
|
182
|
+
if (isCpuWeightBuffer(current)) return current;
|
|
132
183
|
|
|
133
184
|
if (isWeightBuffer(current)) {
|
|
134
185
|
const dtype = getWeightDtype(current);
|
|
@@ -168,7 +219,7 @@ async function maybePromoteEmbeddingsToF32(ctx, current, name, loc) {
|
|
|
168
219
|
|
|
169
220
|
async function maybeDowncastEmbeddings(ctx, current, name, loc) {
|
|
170
221
|
// Can't downcast Float32Array or CpuWeightBuffer
|
|
171
|
-
if (current instanceof Float32Array) {
|
|
222
|
+
if (current instanceof Float32Array || isCpuWeightBuffer(current)) {
|
|
172
223
|
return current;
|
|
173
224
|
}
|
|
174
225
|
|
|
@@ -10,6 +10,7 @@ import { maybeDowncastToF16 } from './weight-downcast.js';
|
|
|
10
10
|
import { getTensorNamesByRole } from './tensors/tensor-role.js';
|
|
11
11
|
import { log, trace as debugTrace } from '../debug/index.js';
|
|
12
12
|
import { selectRuleValue } from '../rules/rule-registry.js';
|
|
13
|
+
import { loadTensorRange } from './tensors/tensor-reader.js';
|
|
13
14
|
|
|
14
15
|
// ============================================================================
|
|
15
16
|
// Constants
|
|
@@ -25,6 +26,34 @@ function isGpuBufferInstance(value) {
|
|
|
25
26
|
return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
|
|
26
27
|
}
|
|
27
28
|
|
|
29
|
+
function createRangeBackedTensorSource(ctx, name, location) {
|
|
30
|
+
if (typeof ctx.loadShardRange !== 'function') {
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
const normalizedLocationDtype = typeof location?.dtype === 'string'
|
|
34
|
+
? location.dtype.toLowerCase()
|
|
35
|
+
: 'f32';
|
|
36
|
+
return {
|
|
37
|
+
kind: 'tensor_range_source',
|
|
38
|
+
sourceDtype: normalizedLocationDtype,
|
|
39
|
+
async loadRange(byteOffset, byteLength) {
|
|
40
|
+
return loadTensorRange(location, name, byteOffset, byteLength, ctx.loadShardRange);
|
|
41
|
+
},
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function createRangeBackedWeightBuffer(ctx, name, location) {
|
|
46
|
+
const source = createRangeBackedTensorSource(ctx, name, location);
|
|
47
|
+
if (!source || !location?.shape || location.shape.length !== 2) {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
const layout = ctx.resolveWeightLayout(location);
|
|
51
|
+
const dtype = selectRuleValue('loader', 'weights', 'floatLocationDtype', {
|
|
52
|
+
locationDtype: location.dtype,
|
|
53
|
+
});
|
|
54
|
+
return createCpuWeightBuffer(source, dtype, layout, location.shape, name);
|
|
55
|
+
}
|
|
56
|
+
|
|
28
57
|
function isLikelyFinalNormName(name) {
|
|
29
58
|
const lower = String(name || '').toLowerCase();
|
|
30
59
|
if (!lower) return false;
|
|
@@ -157,15 +186,20 @@ async function loadLmHead(ctx) {
|
|
|
157
186
|
if (!loc) continue;
|
|
158
187
|
|
|
159
188
|
const shouldStream = ctx.shouldStreamLargeWeight(name, loc, 'LM head');
|
|
160
|
-
const tensor =
|
|
189
|
+
const tensor = shouldStream
|
|
190
|
+
? (
|
|
191
|
+
createRangeBackedWeightBuffer(ctx, name, loc)
|
|
192
|
+
?? await ctx.loadTensor(name, false, true)
|
|
193
|
+
)
|
|
194
|
+
: await ctx.loadTensor(name, true, true);
|
|
161
195
|
|
|
162
|
-
if (shouldStream && tensor && !(tensor instanceof Float32Array)) {
|
|
196
|
+
if (shouldStream && tensor && !(tensor instanceof Float32Array) && !isCpuWeightBuffer(tensor)) {
|
|
163
197
|
throw new Error(
|
|
164
198
|
`[Loader] LM head "${name}" too large for GPU and cannot be loaded on CPU (dtype=${loc.dtype}).`
|
|
165
199
|
);
|
|
166
200
|
}
|
|
167
201
|
|
|
168
|
-
if (tensor && (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || tensor instanceof Float32Array)) {
|
|
202
|
+
if (tensor && (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || isCpuWeightBuffer(tensor) || tensor instanceof Float32Array)) {
|
|
169
203
|
lmHeadName = name;
|
|
170
204
|
lmHeadLoc = loc;
|
|
171
205
|
lmHead = processLmHeadTensor(ctx, tensor, name, loc, shouldStream);
|
|
@@ -253,6 +287,11 @@ async function loadCpuFloatTensor(ctx, tensorName, label) {
|
|
|
253
287
|
|
|
254
288
|
|
|
255
289
|
function processLmHeadTensor(ctx, tensor, name, loc, shouldStream) {
|
|
290
|
+
if (isCpuWeightBuffer(tensor)) {
|
|
291
|
+
log.warn('Loader', `LM head stored on CPU via range-backed source (layout=${tensor.layout})`);
|
|
292
|
+
return tensor;
|
|
293
|
+
}
|
|
294
|
+
|
|
256
295
|
// Float32Array streaming path
|
|
257
296
|
if (tensor instanceof Float32Array && shouldStream) {
|
|
258
297
|
const layout = ctx.resolveWeightLayout(loc);
|
|
@@ -292,8 +331,8 @@ async function maybeDowncastLmHead(ctx, lmHead, lmHeadName, lmHeadLoc) {
|
|
|
292
331
|
return lmHead;
|
|
293
332
|
}
|
|
294
333
|
|
|
295
|
-
// Can't downcast Float32Array
|
|
296
|
-
if (lmHead instanceof Float32Array) {
|
|
334
|
+
// Can't downcast Float32Array or CpuWeightBuffer
|
|
335
|
+
if (lmHead instanceof Float32Array || isCpuWeightBuffer(lmHead)) {
|
|
297
336
|
return lmHead;
|
|
298
337
|
}
|
|
299
338
|
|