doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -8,6 +8,8 @@ import {
|
|
|
8
8
|
openModelStore,
|
|
9
9
|
verifyIntegrity,
|
|
10
10
|
loadManifestFromStore,
|
|
11
|
+
loadAuxFile,
|
|
12
|
+
computeHash,
|
|
11
13
|
} from '../storage/shard-manager.js';
|
|
12
14
|
import { clearManifest, parseManifest, setManifest as setCurrentManifest } from '../formats/rdrr/index.js';
|
|
13
15
|
import { initDevice, getDevice, getKernelCapabilities } from '../gpu/device.js';
|
|
@@ -39,12 +41,15 @@ import { MemoryMonitor } from './memory-monitor.js';
|
|
|
39
41
|
import {
|
|
40
42
|
loadTensorToGPU,
|
|
41
43
|
loadTensorToCPU,
|
|
44
|
+
isLiteRTAffineInt4FusedEligible,
|
|
42
45
|
} from './tensors/tensor-loader.js';
|
|
43
46
|
import { annotateWeightLoadError } from '../inference/pipelines/text/load-errors.js';
|
|
44
47
|
import { loadEmbeddings } from './embedding-loader.js';
|
|
45
48
|
import { loadPerLayerInputWeights } from './per-layer-input-loader.js';
|
|
46
49
|
import { loadLayer } from './layer-loader.js';
|
|
47
50
|
import { loadFinalWeights } from './final-weights-loader.js';
|
|
51
|
+
import { cloneJsonValue } from '../utils/clone-json.js';
|
|
52
|
+
import { assertFunctionalDescriptorManifest } from '../formats/rdrr/functional-descriptor.js';
|
|
48
53
|
import {
|
|
49
54
|
loadExpert as loadExpertFromModule,
|
|
50
55
|
prefetchExperts as prefetchExpertsFromModule,
|
|
@@ -96,6 +101,71 @@ function isGpuBufferInstance(value) {
|
|
|
96
101
|
return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
|
|
97
102
|
}
|
|
98
103
|
|
|
104
|
+
function nowMs() {
|
|
105
|
+
return typeof performance !== 'undefined' && typeof performance.now === 'function'
|
|
106
|
+
? performance.now()
|
|
107
|
+
: Date.now();
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function roundLoadTimingMs(value) {
|
|
111
|
+
return Number.isFinite(value) ? Number(value.toFixed(3)) : null;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function createLoadTiming(modelId, hasCustomLoader) {
|
|
115
|
+
return {
|
|
116
|
+
schemaVersion: 1,
|
|
117
|
+
source: 'doppler-loader',
|
|
118
|
+
modelId: typeof modelId === 'string' ? modelId : null,
|
|
119
|
+
status: 'running',
|
|
120
|
+
customShardLoader: hasCustomLoader === true,
|
|
121
|
+
byteAccountingMode: hasCustomLoader === true
|
|
122
|
+
? 'custom-loader-progress-unavailable'
|
|
123
|
+
: 'full-shard-progress',
|
|
124
|
+
totalBytes: null,
|
|
125
|
+
totalShards: null,
|
|
126
|
+
bytesLoaded: 0,
|
|
127
|
+
shardsLoaded: 0,
|
|
128
|
+
bytesPerSecond: null,
|
|
129
|
+
phasesMs: {
|
|
130
|
+
preflight: null,
|
|
131
|
+
tensorLocations: null,
|
|
132
|
+
embeddings: null,
|
|
133
|
+
layers: null,
|
|
134
|
+
finalWeights: null,
|
|
135
|
+
cleanup: null,
|
|
136
|
+
},
|
|
137
|
+
layers: {
|
|
138
|
+
count: null,
|
|
139
|
+
totalMs: null,
|
|
140
|
+
meanMs: null,
|
|
141
|
+
maxMs: null,
|
|
142
|
+
maxLayer: null,
|
|
143
|
+
},
|
|
144
|
+
totalMs: null,
|
|
145
|
+
failedPhase: null,
|
|
146
|
+
error: null,
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function finishLoadPhase(loadTiming, phase, startMs) {
|
|
151
|
+
if (!loadTiming?.phasesMs || !phase) return;
|
|
152
|
+
loadTiming.phasesMs[phase] = roundLoadTimingMs(nowMs() - startMs);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function finishLoadTiming(loadTiming, status, startMs, error = null, failedPhase = null) {
|
|
156
|
+
if (!loadTiming) return;
|
|
157
|
+
const totalMs = nowMs() - startMs;
|
|
158
|
+
loadTiming.status = status;
|
|
159
|
+
loadTiming.totalMs = roundLoadTimingMs(totalMs);
|
|
160
|
+
loadTiming.bytesPerSecond = totalMs > 0 && Number.isFinite(loadTiming.bytesLoaded)
|
|
161
|
+
? Math.round((loadTiming.bytesLoaded / totalMs) * 1000)
|
|
162
|
+
: null;
|
|
163
|
+
if (status === 'failed') {
|
|
164
|
+
loadTiming.failedPhase = failedPhase;
|
|
165
|
+
loadTiming.error = error?.message ?? String(error ?? 'unknown load error');
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
99
169
|
function getTensorShardIndices(location) {
|
|
100
170
|
if (Array.isArray(location?.spans) && location.spans.length > 0) {
|
|
101
171
|
return Array.from(new Set(location.spans.map((span) => span?.shardIndex).filter(Number.isInteger)));
|
|
@@ -158,6 +228,8 @@ export class DopplerLoader {
|
|
|
158
228
|
|
|
159
229
|
embeddingPostprocessor = null;
|
|
160
230
|
|
|
231
|
+
diffusionGemmaSelfConditioning = null;
|
|
232
|
+
|
|
161
233
|
perLayerInputWeights = null;
|
|
162
234
|
|
|
163
235
|
// Memory management
|
|
@@ -176,6 +248,8 @@ export class DopplerLoader {
|
|
|
176
248
|
|
|
177
249
|
tensorLocations = new Map();
|
|
178
250
|
|
|
251
|
+
loadTiming = null;
|
|
252
|
+
|
|
179
253
|
// Shard cache (LRU with request deduplication)
|
|
180
254
|
|
|
181
255
|
shardCache;
|
|
@@ -186,6 +260,7 @@ export class DopplerLoader {
|
|
|
186
260
|
#loaderDebug = null;
|
|
187
261
|
|
|
188
262
|
#perLayerInputSession = null;
|
|
263
|
+
#loadAuxiliaryFile = null;
|
|
189
264
|
|
|
190
265
|
// Fused Q4_K matmul: skip dequantization for matmul weights, use fused kernel
|
|
191
266
|
|
|
@@ -315,6 +390,13 @@ export class DopplerLoader {
|
|
|
315
390
|
loadRange: options.loadShardRange ?? null,
|
|
316
391
|
streamRange: options.streamShardRange ?? null,
|
|
317
392
|
});
|
|
393
|
+
this.#loadAuxiliaryFile = typeof options.loadAuxiliaryFile === 'function'
|
|
394
|
+
? options.loadAuxiliaryFile
|
|
395
|
+
: null;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
setAuxiliaryFileLoader(loadAuxiliaryFile) {
|
|
399
|
+
this.#loadAuxiliaryFile = typeof loadAuxiliaryFile === 'function' ? loadAuxiliaryFile : null;
|
|
318
400
|
}
|
|
319
401
|
|
|
320
402
|
|
|
@@ -452,6 +534,10 @@ export class DopplerLoader {
|
|
|
452
534
|
|
|
453
535
|
log.info('Loader', `Loading: ${modelId}`);
|
|
454
536
|
this.modelId = modelId;
|
|
537
|
+
const loadTimingStart = nowMs();
|
|
538
|
+
let activeLoadPhase = 'preflight';
|
|
539
|
+
let phaseStart = loadTimingStart;
|
|
540
|
+
this.loadTiming = createLoadTiming(modelId, this.shardCache.hasCustomLoader);
|
|
455
541
|
|
|
456
542
|
this.#startMemoryLogging();
|
|
457
543
|
this.#assertResidentBudget('load start');
|
|
@@ -488,10 +574,17 @@ export class DopplerLoader {
|
|
|
488
574
|
}
|
|
489
575
|
}
|
|
490
576
|
|
|
491
|
-
await this.#buildTensorLocations();
|
|
492
|
-
|
|
493
577
|
const totalBytes = (this.manifest.shards || []).reduce((sum, s) => sum + (s.size || 0), 0);
|
|
494
578
|
const totalShards = this.manifest.shards?.length || 0;
|
|
579
|
+
this.loadTiming.totalBytes = totalBytes;
|
|
580
|
+
this.loadTiming.totalShards = totalShards;
|
|
581
|
+
finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
|
|
582
|
+
|
|
583
|
+
activeLoadPhase = 'tensorLocations';
|
|
584
|
+
phaseStart = nowMs();
|
|
585
|
+
await this.#buildTensorLocations();
|
|
586
|
+
finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
|
|
587
|
+
|
|
495
588
|
const loadStartTime = Date.now();
|
|
496
589
|
let bytesLoaded = 0;
|
|
497
590
|
let shardsLoaded = 0;
|
|
@@ -549,6 +642,8 @@ export class DopplerLoader {
|
|
|
549
642
|
loadedShardIndices.add(shardIndex);
|
|
550
643
|
bytesLoaded += shardSize;
|
|
551
644
|
shardsLoaded++;
|
|
645
|
+
this.loadTiming.bytesLoaded = bytesLoaded;
|
|
646
|
+
this.loadTiming.shardsLoaded = shardsLoaded;
|
|
552
647
|
if (!inLayerPhase) {
|
|
553
648
|
const pct = 0.1 + Math.min(bytesLoaded / totalBytes, 1.0) * 0.7;
|
|
554
649
|
const elapsed = (Date.now() - loadStartTime) / 1000;
|
|
@@ -582,7 +677,10 @@ export class DopplerLoader {
|
|
|
582
677
|
let loadError = null;
|
|
583
678
|
try {
|
|
584
679
|
reportProgress('shards', 0.1, 'Loading embeddings...');
|
|
680
|
+
activeLoadPhase = 'embeddings';
|
|
681
|
+
phaseStart = nowMs();
|
|
585
682
|
await this.#loadEmbeddings(onProgress);
|
|
683
|
+
finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
|
|
586
684
|
this.#assertResidentBudget('embeddings');
|
|
587
685
|
|
|
588
686
|
const resolveNumLayers = (value) => {
|
|
@@ -616,14 +714,24 @@ export class DopplerLoader {
|
|
|
616
714
|
log.info('Loader', `Layers: 0-${numLayers - 1}`);
|
|
617
715
|
|
|
618
716
|
inLayerPhase = true;
|
|
717
|
+
activeLoadPhase = 'layers';
|
|
619
718
|
const layersStartTime = performance.now();
|
|
719
|
+
let layerTotalMs = 0;
|
|
720
|
+
let maxLayerMs = 0;
|
|
721
|
+
let maxLayer = null;
|
|
620
722
|
|
|
621
723
|
for (let l = 0; l < numLayers; l++) {
|
|
622
724
|
const layerStart = performance.now();
|
|
623
725
|
const layerPromise = this.#loadLayer(l, onProgress);
|
|
624
726
|
this.#prefetchLayerShards(l);
|
|
625
727
|
await layerPromise;
|
|
626
|
-
const
|
|
728
|
+
const layerElapsedMs = performance.now() - layerStart;
|
|
729
|
+
layerTotalMs += layerElapsedMs;
|
|
730
|
+
if (layerElapsedMs > maxLayerMs) {
|
|
731
|
+
maxLayerMs = layerElapsedMs;
|
|
732
|
+
maxLayer = l;
|
|
733
|
+
}
|
|
734
|
+
const layerElapsed = (layerElapsedMs / 1000).toFixed(2);
|
|
627
735
|
log.verbose('Loader', ` Layer ${l}: ${layerElapsed}s`);
|
|
628
736
|
|
|
629
737
|
await new Promise(r => setTimeout(r, 0));
|
|
@@ -661,10 +769,21 @@ export class DopplerLoader {
|
|
|
661
769
|
}
|
|
662
770
|
|
|
663
771
|
const layersTotalTime = ((performance.now() - layersStartTime) / 1000).toFixed(2);
|
|
772
|
+
this.loadTiming.layers = {
|
|
773
|
+
count: numLayers,
|
|
774
|
+
totalMs: roundLoadTimingMs(layerTotalMs),
|
|
775
|
+
meanMs: numLayers > 0 ? roundLoadTimingMs(layerTotalMs / numLayers) : null,
|
|
776
|
+
maxMs: maxLayer == null ? null : roundLoadTimingMs(maxLayerMs),
|
|
777
|
+
maxLayer,
|
|
778
|
+
};
|
|
779
|
+
finishLoadPhase(this.loadTiming, activeLoadPhase, layersStartTime);
|
|
664
780
|
log.info('Loader', `Layers: ${numLayers} complete (${layersTotalTime}s)`);
|
|
665
781
|
|
|
666
782
|
reportProgress('gpu_transfer', 0.85, 'Loading final weights...');
|
|
783
|
+
activeLoadPhase = 'finalWeights';
|
|
784
|
+
phaseStart = nowMs();
|
|
667
785
|
await this.#loadFinalWeights(onProgress);
|
|
786
|
+
finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
|
|
668
787
|
this.#assertResidentBudget('final weights');
|
|
669
788
|
|
|
670
789
|
if (onProgress) {
|
|
@@ -676,11 +795,16 @@ export class DopplerLoader {
|
|
|
676
795
|
const avgSpeed = formatBytes(bytesLoaded / (Date.now() - loadStartTime) * 1000);
|
|
677
796
|
log.info('Loader', `Complete: ${formatBytes(bytesLoaded)} in ${totalTime}s (${avgSpeed}/s)`);
|
|
678
797
|
|
|
798
|
+
activeLoadPhase = 'cleanup';
|
|
799
|
+
phaseStart = nowMs();
|
|
679
800
|
this.shardCache.clear();
|
|
801
|
+
finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
|
|
802
|
+
finishLoadTiming(this.loadTiming, 'complete', loadTimingStart);
|
|
680
803
|
|
|
681
804
|
return (this.manifest.config) || {};
|
|
682
805
|
} catch (error) {
|
|
683
806
|
loadError = error;
|
|
807
|
+
finishLoadTiming(this.loadTiming, 'failed', loadTimingStart, error, activeLoadPhase);
|
|
684
808
|
} finally {
|
|
685
809
|
this.#loadShardOverride = null;
|
|
686
810
|
if (this.#memoryMonitor) {
|
|
@@ -734,12 +858,8 @@ export class DopplerLoader {
|
|
|
734
858
|
this.#layerShardMap.set(layerIdx, shards);
|
|
735
859
|
}
|
|
736
860
|
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
shards.add(span.shardIndex);
|
|
740
|
-
}
|
|
741
|
-
} else {
|
|
742
|
-
shards.add(location.shardIndex);
|
|
861
|
+
for (const shardIndex of getTensorShardIndices(location)) {
|
|
862
|
+
shards.add(shardIndex);
|
|
743
863
|
}
|
|
744
864
|
}
|
|
745
865
|
}
|
|
@@ -827,9 +947,16 @@ export class DopplerLoader {
|
|
|
827
947
|
const streamedUpload = toGPU && this.#shouldStreamUploadToGPU(location);
|
|
828
948
|
let shardData;
|
|
829
949
|
try {
|
|
830
|
-
|
|
950
|
+
const preserveRawSourceBytes = toGPU && isLiteRTAffineInt4FusedEligible(location, {
|
|
951
|
+
gpuCapabilities: this.gpuCapabilities,
|
|
952
|
+
});
|
|
953
|
+
shardData = this.#isFunctionalDescriptorLocation(location)
|
|
954
|
+
? await this.#assembleFunctionalDescriptorData(location, name)
|
|
955
|
+
: streamedUpload
|
|
831
956
|
? await this.#assembleShardDataToGpuBuffer(location, name)
|
|
832
|
-
: await this.#assembleShardData(location, name
|
|
957
|
+
: await this.#assembleShardData(location, name, {
|
|
958
|
+
materializeSourceTransform: !preserveRawSourceBytes,
|
|
959
|
+
});
|
|
833
960
|
} catch (error) {
|
|
834
961
|
throw annotateTensorLoadError(error, name, location, {
|
|
835
962
|
tensorLoadStage: streamedUpload ? 'streamShardToGpuBuffer' : 'assembleShardData',
|
|
@@ -846,7 +973,7 @@ export class DopplerLoader {
|
|
|
846
973
|
releaseBuffer(shardData);
|
|
847
974
|
shardData = await this.#assembleShardData(location, name);
|
|
848
975
|
}
|
|
849
|
-
return loadTensorToCPU(shardData, location);
|
|
976
|
+
return loadTensorToCPU(shardData, location, name);
|
|
850
977
|
}
|
|
851
978
|
|
|
852
979
|
|
|
@@ -891,19 +1018,145 @@ export class DopplerLoader {
|
|
|
891
1018
|
releaseBuffer(shardData);
|
|
892
1019
|
shardData = await this.#assembleShardData(location, name);
|
|
893
1020
|
}
|
|
894
|
-
return loadTensorToCPU(shardData, location);
|
|
1021
|
+
return loadTensorToCPU(shardData, location, name);
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
#isFunctionalDescriptorLocation(location) {
|
|
1025
|
+
return String(location?.dtype || '').trim().toUpperCase() === 'FUNCTIONAL_DESCRIPTOR';
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
#getDescriptorShardFiles(location, name) {
|
|
1029
|
+
if (!location?.descriptorManifest) {
|
|
1030
|
+
throw new Error(
|
|
1031
|
+
`[DopplerLoader] FUNCTIONAL_DESCRIPTOR tensor "${name}" is missing descriptorManifest.`
|
|
1032
|
+
);
|
|
1033
|
+
}
|
|
1034
|
+
const manifest = assertFunctionalDescriptorManifest(
|
|
1035
|
+
location.descriptorManifest,
|
|
1036
|
+
`FUNCTIONAL_DESCRIPTOR tensor "${name}" descriptorManifest`
|
|
1037
|
+
);
|
|
1038
|
+
const components = manifest.components;
|
|
1039
|
+
if (!components || typeof components !== 'object') {
|
|
1040
|
+
throw new Error(
|
|
1041
|
+
`[DopplerLoader] FUNCTIONAL_DESCRIPTOR tensor "${name}" descriptorManifest.components is required.`
|
|
1042
|
+
);
|
|
1043
|
+
}
|
|
1044
|
+
const files = [
|
|
1045
|
+
components.kronecker_sum?.shard_file,
|
|
1046
|
+
components.coordinate_inr?.shard_file,
|
|
1047
|
+
components.sparse_outliers?.shard_file,
|
|
1048
|
+
];
|
|
1049
|
+
if (files.some((file) => typeof file !== 'string' || file.trim().length === 0)) {
|
|
1050
|
+
throw new Error(
|
|
1051
|
+
`[DopplerLoader] FUNCTIONAL_DESCRIPTOR tensor "${name}" must declare kronecker, SIREN, and sparse shard_file values.`
|
|
1052
|
+
);
|
|
1053
|
+
}
|
|
1054
|
+
return files.map((file) => file.trim());
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
async #loadDescriptorShardFile(file, name) {
|
|
1058
|
+
const loadAuxiliaryFile = this.#loadAuxiliaryFile;
|
|
1059
|
+
const payload = loadAuxiliaryFile
|
|
1060
|
+
? await loadAuxiliaryFile(file)
|
|
1061
|
+
: await loadAuxFile(file);
|
|
1062
|
+
if (payload == null) {
|
|
1063
|
+
throw new Error(
|
|
1064
|
+
`[DopplerLoader] Descriptor shard "${file}" for tensor "${name}" was not found.`
|
|
1065
|
+
);
|
|
1066
|
+
}
|
|
1067
|
+
if (payload instanceof Uint8Array) {
|
|
1068
|
+
return payload;
|
|
1069
|
+
}
|
|
1070
|
+
if (ArrayBuffer.isView(payload)) {
|
|
1071
|
+
return new Uint8Array(payload.buffer, payload.byteOffset, payload.byteLength);
|
|
1072
|
+
}
|
|
1073
|
+
if (payload instanceof ArrayBuffer) {
|
|
1074
|
+
return new Uint8Array(payload);
|
|
1075
|
+
}
|
|
1076
|
+
throw new Error(
|
|
1077
|
+
`[DopplerLoader] Descriptor shard "${file}" for tensor "${name}" must load as ArrayBuffer or Uint8Array.`
|
|
1078
|
+
);
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
async #assertDescriptorHash(location, name, descriptorShards) {
|
|
1082
|
+
const descriptorHash = location?.descriptorManifest?.descriptor_hash;
|
|
1083
|
+
if (typeof descriptorHash !== 'string' || !descriptorHash.trim()) {
|
|
1084
|
+
return;
|
|
1085
|
+
}
|
|
1086
|
+
const match = /^sha256:([a-f0-9]{64})$/i.exec(descriptorHash.trim());
|
|
1087
|
+
if (!match) {
|
|
1088
|
+
throw new Error(
|
|
1089
|
+
`[DopplerLoader] FUNCTIONAL_DESCRIPTOR tensor "${name}" descriptor_hash must be sha256:<64 hex chars>.`
|
|
1090
|
+
);
|
|
1091
|
+
}
|
|
1092
|
+
const totalBytes = Array.from(descriptorShards.values()).reduce((sum, bytes) => sum + bytes.byteLength, 0);
|
|
1093
|
+
const combined = new Uint8Array(totalBytes);
|
|
1094
|
+
let offset = 0;
|
|
1095
|
+
for (const bytes of descriptorShards.values()) {
|
|
1096
|
+
combined.set(bytes, offset);
|
|
1097
|
+
offset += bytes.byteLength;
|
|
1098
|
+
}
|
|
1099
|
+
const actual = await computeHash(combined, 'sha256');
|
|
1100
|
+
if (actual.toLowerCase() !== match[1].toLowerCase()) {
|
|
1101
|
+
throw new Error(
|
|
1102
|
+
`[DopplerLoader] FUNCTIONAL_DESCRIPTOR tensor "${name}" descriptor hash mismatch. ` +
|
|
1103
|
+
`Expected ${descriptorHash}, got sha256:${actual}.`
|
|
1104
|
+
);
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
async #assembleFunctionalDescriptorData(location, name) {
|
|
1109
|
+
const shardFiles = this.#getDescriptorShardFiles(location, name);
|
|
1110
|
+
const descriptorShards = new Map();
|
|
1111
|
+
for (const file of shardFiles) {
|
|
1112
|
+
descriptorShards.set(file, await this.#loadDescriptorShardFile(file, name));
|
|
1113
|
+
}
|
|
1114
|
+
await this.#assertDescriptorHash(location, name, descriptorShards);
|
|
1115
|
+
const data = new Uint8Array(0);
|
|
1116
|
+
Object.defineProperty(data, 'descriptorShards', {
|
|
1117
|
+
value: descriptorShards,
|
|
1118
|
+
enumerable: false,
|
|
1119
|
+
});
|
|
1120
|
+
return data;
|
|
895
1121
|
}
|
|
896
1122
|
|
|
897
1123
|
|
|
898
|
-
async #assembleShardData(location, name) {
|
|
1124
|
+
async #assembleShardData(location, name, options = {}) {
|
|
899
1125
|
const loadShard = this.#getLoadShard();
|
|
900
1126
|
const loadShardRange = (idx, offset, length) => this.shardCache.loadRange(idx, offset, length);
|
|
901
|
-
|
|
1127
|
+
const data = await assembleShardData(location, name, loadShard, loadShardRange, options);
|
|
1128
|
+
const companions = Array.isArray(location?.storage?.companions)
|
|
1129
|
+
? location.storage.companions
|
|
1130
|
+
: [];
|
|
1131
|
+
if (companions.length === 0) {
|
|
1132
|
+
return data;
|
|
1133
|
+
}
|
|
1134
|
+
const storageCompanions = {};
|
|
1135
|
+
for (const companion of companions) {
|
|
1136
|
+
const companionLocation = this.tensorLocations.get(companion.tensorId);
|
|
1137
|
+
if (!companionLocation) {
|
|
1138
|
+
throw new Error(
|
|
1139
|
+
`[DopplerLoader] Tensor "${name}" storage companion "${companion.tensorId}" for role "${companion.role}" was not found.`
|
|
1140
|
+
);
|
|
1141
|
+
}
|
|
1142
|
+
storageCompanions[companion.role] = {
|
|
1143
|
+
tensorId: companion.tensorId,
|
|
1144
|
+
location: companionLocation,
|
|
1145
|
+
bytes: await assembleShardData(companionLocation, companion.tensorId, loadShard, loadShardRange),
|
|
1146
|
+
};
|
|
1147
|
+
}
|
|
1148
|
+
Object.defineProperty(data, 'storageCompanions', {
|
|
1149
|
+
value: storageCompanions,
|
|
1150
|
+
enumerable: false,
|
|
1151
|
+
});
|
|
1152
|
+
return data;
|
|
902
1153
|
}
|
|
903
1154
|
|
|
904
1155
|
#shouldStreamUploadToGPU(location) {
|
|
1156
|
+
if (this.#isFunctionalDescriptorLocation(location)) return false;
|
|
905
1157
|
if (!location?.size || location.size <= 0) return false;
|
|
906
1158
|
if (hasSourceTransform(location)) return false;
|
|
1159
|
+
if (Array.isArray(location?.storage?.companions) && location.storage.companions.length > 0) return false;
|
|
907
1160
|
if (this.shardCache.hasCustomLoader && !this.shardCache.canStreamRanges) return false;
|
|
908
1161
|
const chunkBytes = this.#loadingConfig?.storage?.backend?.streaming?.readChunkBytes ?? 0;
|
|
909
1162
|
if (!Number.isFinite(chunkBytes) || chunkBytes <= 0) return false;
|
|
@@ -1015,6 +1268,7 @@ export class DopplerLoader {
|
|
|
1015
1268
|
// gather.wgsl reads embeddings as f32; downcasting here corrupts reads.
|
|
1016
1269
|
preserveF32Embeddings: String(this.manifest?.quantizationInfo?.embeddings ?? '').toLowerCase() === 'f32',
|
|
1017
1270
|
hostHasShaderF16: this.gpuCapabilities?.hasF16 ?? null,
|
|
1271
|
+
embeddingKernel: this.manifest?.inference?.execution?.kernels?.embed ?? null,
|
|
1018
1272
|
};
|
|
1019
1273
|
|
|
1020
1274
|
this.embeddings = await loadEmbeddings(ctx);
|
|
@@ -1049,6 +1303,7 @@ export class DopplerLoader {
|
|
|
1049
1303
|
keepF32Weights: this.keepF32Weights,
|
|
1050
1304
|
isMoE: this.isMoE,
|
|
1051
1305
|
isExpertLayer: (idx) => this.#isExpertLayer(idx),
|
|
1306
|
+
loadDenseFfnForMoeLayers: this.manifest?.inference?.ffn?.branchMode === 'dense_plus_moe',
|
|
1052
1307
|
numHeads: this.manifest?.architecture?.numAttentionHeads ?? null,
|
|
1053
1308
|
numKVHeads: this.manifest?.architecture?.numKeyValueHeads ?? null,
|
|
1054
1309
|
headDim: this.manifest?.architecture?.headDim ?? null,
|
|
@@ -1088,6 +1343,7 @@ export class DopplerLoader {
|
|
|
1088
1343
|
const loadShard = this.#getLoadShard();
|
|
1089
1344
|
return {
|
|
1090
1345
|
manifest: this.manifest,
|
|
1346
|
+
tensorLocations: this.tensorLocations,
|
|
1091
1347
|
loadTensor: (name, toGPU, silent) => this.#loadTensor(name, toGPU, silent),
|
|
1092
1348
|
loadShard,
|
|
1093
1349
|
shardCache: this.shardCache,
|
|
@@ -1119,6 +1375,7 @@ export class DopplerLoader {
|
|
|
1119
1375
|
resolveWeightLayout: (loc) => this.#resolveWeightLayout(loc),
|
|
1120
1376
|
embeddings: this.embeddings,
|
|
1121
1377
|
embeddingPostprocessor: this.manifest?.inference?.output?.embeddingPostprocessor ?? null,
|
|
1378
|
+
diffusionGemmaSelfConditioning: this.manifest?.inference?.diffusionGemma?.selfConditioning === true,
|
|
1122
1379
|
modelType: this.manifest?.modelType ?? null,
|
|
1123
1380
|
tieWordEmbeddings,
|
|
1124
1381
|
gpuBuffers: this.gpuBuffers,
|
|
@@ -1130,6 +1387,7 @@ export class DopplerLoader {
|
|
|
1130
1387
|
this.finalNorm = result.finalNorm;
|
|
1131
1388
|
this.lmHead = result.lmHead;
|
|
1132
1389
|
this.embeddingPostprocessor = result.embeddingPostprocessor;
|
|
1390
|
+
this.diffusionGemmaSelfConditioning = result.diffusionGemmaSelfConditioning;
|
|
1133
1391
|
this.#normOffsetDebugLogged = result.normOffsetDebugLogged;
|
|
1134
1392
|
}
|
|
1135
1393
|
|
|
@@ -1166,9 +1424,14 @@ export class DopplerLoader {
|
|
|
1166
1424
|
layersLoaded: this.layers.size,
|
|
1167
1425
|
expertsLoaded: this.experts.size + expertCacheCount,
|
|
1168
1426
|
gpuBuffers: this.gpuBuffers.size,
|
|
1427
|
+
loadTiming: this.getLoadTiming(),
|
|
1169
1428
|
};
|
|
1170
1429
|
}
|
|
1171
1430
|
|
|
1431
|
+
getLoadTiming() {
|
|
1432
|
+
return this.loadTiming ? cloneJsonValue(this.loadTiming) : null;
|
|
1433
|
+
}
|
|
1434
|
+
|
|
1172
1435
|
|
|
1173
1436
|
getExpertCacheStats() {
|
|
1174
1437
|
return this.expertCache?.getStats() || null;
|
|
@@ -1199,10 +1462,21 @@ export class DopplerLoader {
|
|
|
1199
1462
|
}
|
|
1200
1463
|
this.gpuBuffers.delete(gpuBuffer);
|
|
1201
1464
|
};
|
|
1465
|
+
const releaseSplitCandidate = (value) => {
|
|
1466
|
+
const sections = value?.gpuSplitWeight?.sections;
|
|
1467
|
+
if (!Array.isArray(sections)) {
|
|
1468
|
+
return;
|
|
1469
|
+
}
|
|
1470
|
+
for (const section of sections) {
|
|
1471
|
+
releaseCandidate(section?.buffer ?? null);
|
|
1472
|
+
}
|
|
1473
|
+
value.gpuSplitWeight = null;
|
|
1474
|
+
};
|
|
1202
1475
|
|
|
1203
1476
|
for (const buffer of this.gpuBuffers) {
|
|
1204
1477
|
releaseCandidate(buffer);
|
|
1205
1478
|
}
|
|
1479
|
+
releaseSplitCandidate(this.lmHead);
|
|
1206
1480
|
this.gpuBuffers.clear();
|
|
1207
1481
|
|
|
1208
1482
|
if (this.expertCache) {
|
|
@@ -1236,6 +1510,7 @@ export class DopplerLoader {
|
|
|
1236
1510
|
this.modelId = null;
|
|
1237
1511
|
this.loadedShards.clear();
|
|
1238
1512
|
this.isLoaded = false;
|
|
1513
|
+
this.loadTiming = null;
|
|
1239
1514
|
this.tensorLocations.clear();
|
|
1240
1515
|
this.#layerShardMap.clear();
|
|
1241
1516
|
this.shardCache.clear();
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export const MAX_SPLIT4_EMBEDDING_SECTIONS: 4;
|
|
2
|
+
export const MAX_SPLIT8_EMBEDDING_SECTIONS: 8;
|
|
3
|
+
export const MAX_SPLIT_EMBEDDING_SECTIONS: 8;
|
|
4
|
+
export const SPLIT_EMBEDDING_STORAGE_BUFFER_OVERHEAD: 2;
|
|
5
|
+
|
|
6
|
+
export function getEmbeddingFloatDtype(location: unknown): string;
|
|
7
|
+
|
|
8
|
+
export function expectsSplitGpuEmbeddingKernel(embeddingKernel: unknown): boolean;
|
|
9
|
+
|
|
10
|
+
export function getSplitGpuEmbeddingKernelSectionCount(embeddingKernel: unknown): number;
|
|
11
|
+
|
|
12
|
+
export function getSplitGpuEmbeddingRequiredStorageBuffers(sectionCount: number): number;
|
|
13
|
+
|
|
14
|
+
export function getMaxSplitGpuEmbeddingSectionsForDevice(
|
|
15
|
+
embeddingKernel?: unknown,
|
|
16
|
+
device?: unknown
|
|
17
|
+
): number;
|
|
18
|
+
|
|
19
|
+
export function createGpuResidentEmbeddingLimitError(options: {
|
|
20
|
+
name: string;
|
|
21
|
+
location: unknown;
|
|
22
|
+
embeddingKernel?: unknown;
|
|
23
|
+
}): Error | null;
|
|
24
|
+
|
|
25
|
+
export function resolveManifestGpuResidentEmbeddingLimitError(
|
|
26
|
+
manifest: unknown,
|
|
27
|
+
options?: {
|
|
28
|
+
storageManifest?: unknown;
|
|
29
|
+
runtimeConfig?: unknown;
|
|
30
|
+
}
|
|
31
|
+
): Error | null;
|