doppler-gpu 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +67 -34
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.js +5 -1
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/version.js +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
|
|
2
|
-
|
|
3
2
|
import { getDevice } from '../../gpu/device.js';
|
|
3
|
+
import { recordKVCacheWriteF32ToF16 } from '../../gpu/kernel-selector.js';
|
|
4
4
|
import { allowReadback } from '../../gpu/perf-guards.js';
|
|
5
5
|
import { log } from '../../debug/index.js';
|
|
6
6
|
import { readBuffer } from '../../memory/buffer-pool.js';
|
|
@@ -473,6 +473,114 @@ export class KVCache {
|
|
|
473
473
|
this.currentSeqLen = Math.max(this.currentSeqLen, startPos + numTokens);
|
|
474
474
|
}
|
|
475
475
|
|
|
476
|
+
|
|
477
|
+
async recordUpdateF32ToF16FromGPU(
|
|
478
|
+
recorder,
|
|
479
|
+
layerIdx,
|
|
480
|
+
keysBuffer,
|
|
481
|
+
valuesBuffer,
|
|
482
|
+
startPos,
|
|
483
|
+
numTokens
|
|
484
|
+
) {
|
|
485
|
+
this._assertLayerIndex(layerIdx);
|
|
486
|
+
this._assertStartPos(startPos);
|
|
487
|
+
if (this.kvDtype !== 'f16') {
|
|
488
|
+
throw new Error('KVCache recordUpdateF32ToF16FromGPU requires an f16 KV cache.');
|
|
489
|
+
}
|
|
490
|
+
if (!Number.isInteger(numTokens) || numTokens < 0) {
|
|
491
|
+
throw new Error('KVCache recordUpdateF32ToF16FromGPU requires a non-negative integer token count.');
|
|
492
|
+
}
|
|
493
|
+
if (numTokens === 0) {
|
|
494
|
+
return;
|
|
495
|
+
}
|
|
496
|
+
this.counters.recordedGpuUpdateCalls += 1;
|
|
497
|
+
this.counters.tokensWritten += numTokens;
|
|
498
|
+
|
|
499
|
+
const layer = (this.layers[layerIdx]);
|
|
500
|
+
if (!layer.keysGPU) {
|
|
501
|
+
throw new Error('GPU cache not initialized');
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
if (startPos + numTokens > this.maxSeqLen) {
|
|
505
|
+
throw new Error(
|
|
506
|
+
`Cache overflow: ${startPos + numTokens} > ${this.maxSeqLen}`
|
|
507
|
+
);
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
const elementCount = numTokens * this.kvSize;
|
|
511
|
+
await recordKVCacheWriteF32ToF16(
|
|
512
|
+
recorder,
|
|
513
|
+
keysBuffer,
|
|
514
|
+
valuesBuffer,
|
|
515
|
+
layer.keysGPU,
|
|
516
|
+
layer.valuesGPU,
|
|
517
|
+
{
|
|
518
|
+
srcOffset: 0,
|
|
519
|
+
dstOffset: startPos * this.kvSize,
|
|
520
|
+
elementCount,
|
|
521
|
+
}
|
|
522
|
+
);
|
|
523
|
+
|
|
524
|
+
if (this.layout === 'paged') {
|
|
525
|
+
const neededPages = Math.ceil((startPos + numTokens) / this.pageSize);
|
|
526
|
+
if (Number.isFinite(layer.allocatedPages)) {
|
|
527
|
+
layer.allocatedPages = Math.max(layer.allocatedPages, neededPages);
|
|
528
|
+
} else {
|
|
529
|
+
layer.allocatedPages = neededPages;
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
layer.seqLen = Math.max(layer.seqLen, startPos + numTokens);
|
|
534
|
+
this.totalTokensSeen = Math.max(this.totalTokensSeen, startPos + numTokens);
|
|
535
|
+
|
|
536
|
+
this.currentSeqLen = Math.max(this.currentSeqLen, startPos + numTokens);
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
recordF16UpdateAlreadyWrittenFromGPU(
|
|
540
|
+
layerIdx,
|
|
541
|
+
startPos,
|
|
542
|
+
numTokens
|
|
543
|
+
) {
|
|
544
|
+
this._assertLayerIndex(layerIdx);
|
|
545
|
+
this._assertStartPos(startPos);
|
|
546
|
+
if (this.kvDtype !== 'f16') {
|
|
547
|
+
throw new Error('KVCache recordF16UpdateAlreadyWrittenFromGPU requires an f16 KV cache.');
|
|
548
|
+
}
|
|
549
|
+
if (this.layout !== 'contiguous') {
|
|
550
|
+
throw new Error('KVCache recordF16UpdateAlreadyWrittenFromGPU requires contiguous layout.');
|
|
551
|
+
}
|
|
552
|
+
if (!Number.isInteger(numTokens) || numTokens < 0) {
|
|
553
|
+
throw new Error('KVCache recordF16UpdateAlreadyWrittenFromGPU requires a non-negative integer token count.');
|
|
554
|
+
}
|
|
555
|
+
if (numTokens === 0) {
|
|
556
|
+
return;
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
const layer = (this.layers[layerIdx]);
|
|
560
|
+
if (!isContiguousLayer(layer) || !layer.keysGPU || !layer.valuesGPU) {
|
|
561
|
+
throw new Error('GPU cache not initialized');
|
|
562
|
+
}
|
|
563
|
+
if (startPos + numTokens > this.maxSeqLen) {
|
|
564
|
+
throw new Error(
|
|
565
|
+
`Cache overflow: ${startPos + numTokens} > ${this.maxSeqLen}`
|
|
566
|
+
);
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
const requiredBytes = (startPos + numTokens) * this.kvSize * this.bytesPerElem;
|
|
570
|
+
if (Number.isFinite(layer.keysGPU.size) && requiredBytes > layer.keysGPU.size) {
|
|
571
|
+
throw new Error('KVCache direct f16 keys write exceeds GPU cache buffer size.');
|
|
572
|
+
}
|
|
573
|
+
if (Number.isFinite(layer.valuesGPU.size) && requiredBytes > layer.valuesGPU.size) {
|
|
574
|
+
throw new Error('KVCache direct f16 values write exceeds GPU cache buffer size.');
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
this.counters.recordedGpuUpdateCalls += 1;
|
|
578
|
+
this.counters.tokensWritten += numTokens;
|
|
579
|
+
layer.seqLen = Math.max(layer.seqLen, startPos + numTokens);
|
|
580
|
+
this.totalTokensSeen = Math.max(this.totalTokensSeen, startPos + numTokens);
|
|
581
|
+
this.currentSeqLen = Math.max(this.currentSeqLen, startPos + numTokens);
|
|
582
|
+
}
|
|
583
|
+
|
|
476
584
|
|
|
477
585
|
_updateContiguous(
|
|
478
586
|
layer,
|
|
@@ -12,6 +12,7 @@ export interface MixedGeometryKVCacheConfig {
|
|
|
12
12
|
pageSize: number;
|
|
13
13
|
kvDtype: 'f16' | 'f32';
|
|
14
14
|
slidingWindow?: number | null;
|
|
15
|
+
slidingLayerLayout: 'ring' | 'contiguous';
|
|
15
16
|
layerTypes: string[];
|
|
16
17
|
}
|
|
17
18
|
|
|
@@ -70,6 +71,15 @@ export declare class MixedGeometryKVCache {
|
|
|
70
71
|
numTokens: number,
|
|
71
72
|
tokenIds?: number[] | null
|
|
72
73
|
): void;
|
|
74
|
+
recordUpdateF32ToF16FromGPU(
|
|
75
|
+
recorder: import('../../gpu/kernel-selector.js').CommandRecorder,
|
|
76
|
+
layerIdx: number,
|
|
77
|
+
keysBuffer: GPUBuffer,
|
|
78
|
+
valuesBuffer: GPUBuffer,
|
|
79
|
+
startPos: number,
|
|
80
|
+
numTokens: number,
|
|
81
|
+
tokenIds?: number[] | null
|
|
82
|
+
): void | Promise<void>;
|
|
73
83
|
get(): never;
|
|
74
84
|
getKeyCache(layerIdx: number): GPUBuffer | null;
|
|
75
85
|
getValueCache(layerIdx: number): GPUBuffer | null;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { getDevice } from '../../gpu/device.js';
|
|
2
|
+
import { recordKVCacheWriteF32ToF16 } from '../../gpu/kernel-selector.js';
|
|
2
3
|
|
|
3
4
|
function normalizeLayerType(layerType) {
|
|
4
5
|
return typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
|
|
@@ -19,6 +20,12 @@ function buildLayerSpecs(config, bytesPerElem) {
|
|
|
19
20
|
'MixedGeometryKVCache requires explicit layerTypes for every layer so per-layer KV geometry stays manifest-owned.'
|
|
20
21
|
);
|
|
21
22
|
}
|
|
23
|
+
const slidingLayerLayout = config.slidingLayerLayout;
|
|
24
|
+
if (slidingLayerLayout !== 'ring' && slidingLayerLayout !== 'contiguous') {
|
|
25
|
+
throw new Error(
|
|
26
|
+
'MixedGeometryKVCache requires slidingLayerLayout to be "ring" or "contiguous".'
|
|
27
|
+
);
|
|
28
|
+
}
|
|
22
29
|
|
|
23
30
|
const baseHeadDim = Number(config.headDim);
|
|
24
31
|
const globalHeadDim = Number.isFinite(config.globalHeadDim) && config.globalHeadDim > 0
|
|
@@ -46,7 +53,10 @@ function buildLayerSpecs(config, bytesPerElem) {
|
|
|
46
53
|
return layerTypes.map((layerType, layerIdx) => {
|
|
47
54
|
const isSliding = isSlidingLayerType(layerType);
|
|
48
55
|
const headDim = isSliding ? baseHeadDim : globalHeadDim;
|
|
49
|
-
const
|
|
56
|
+
const layout = isSliding && slidingWindow != null && slidingLayerLayout === 'ring'
|
|
57
|
+
? 'ring'
|
|
58
|
+
: 'contiguous';
|
|
59
|
+
const capacityTokens = layout === 'ring'
|
|
50
60
|
? slidingWindow
|
|
51
61
|
: maxSeqLen;
|
|
52
62
|
const layerNumHeads = isSliding ? numHeads : globalNumHeads;
|
|
@@ -55,7 +65,7 @@ function buildLayerSpecs(config, bytesPerElem) {
|
|
|
55
65
|
return {
|
|
56
66
|
layerIdx,
|
|
57
67
|
layerType,
|
|
58
|
-
layout
|
|
68
|
+
layout,
|
|
59
69
|
headDim,
|
|
60
70
|
numHeads: layerNumHeads,
|
|
61
71
|
kvSize,
|
|
@@ -154,6 +164,7 @@ export class MixedGeometryKVCache {
|
|
|
154
164
|
globalHeadDim: Number.isFinite(config.globalHeadDim) ? Math.trunc(config.globalHeadDim) : null,
|
|
155
165
|
maxSeqLen: this.maxSeqLen,
|
|
156
166
|
pageSize: this.pageSize,
|
|
167
|
+
slidingLayerLayout: config.slidingLayerLayout,
|
|
157
168
|
};
|
|
158
169
|
this.layerSpecs = buildLayerSpecs(this._config, this.bytesPerElem);
|
|
159
170
|
this.layers = new Array(this.numLayers);
|
|
@@ -206,6 +217,27 @@ export class MixedGeometryKVCache {
|
|
|
206
217
|
encoder.copyBufferToBuffer(valuesBuffer, 0, layer.valuesGPU, byteOffset, byteSize);
|
|
207
218
|
}
|
|
208
219
|
|
|
220
|
+
async _recordContiguousF32ToF16(recorder, layer, spec, keysBuffer, valuesBuffer, startPos, numTokens) {
|
|
221
|
+
const endPos = startPos + numTokens;
|
|
222
|
+
if (endPos > spec.capacityTokens) {
|
|
223
|
+
throw new Error(
|
|
224
|
+
`MixedGeometryKVCache overflow at layer ${spec.layerIdx}: ${endPos} > ${spec.capacityTokens}.`
|
|
225
|
+
);
|
|
226
|
+
}
|
|
227
|
+
await recordKVCacheWriteF32ToF16(
|
|
228
|
+
recorder,
|
|
229
|
+
keysBuffer,
|
|
230
|
+
valuesBuffer,
|
|
231
|
+
layer.keysGPU,
|
|
232
|
+
layer.valuesGPU,
|
|
233
|
+
{
|
|
234
|
+
srcOffset: 0,
|
|
235
|
+
dstOffset: startPos * spec.kvSize,
|
|
236
|
+
elementCount: numTokens * spec.kvSize,
|
|
237
|
+
}
|
|
238
|
+
);
|
|
239
|
+
}
|
|
240
|
+
|
|
209
241
|
_recordRingCopy(encoder, layer, spec, keysBuffer, valuesBuffer, startPos, numTokens) {
|
|
210
242
|
const windowSize = spec.capacityTokens;
|
|
211
243
|
const bytesPerToken = spec.bytesPerToken;
|
|
@@ -245,6 +277,60 @@ export class MixedGeometryKVCache {
|
|
|
245
277
|
layer.seqLen = Math.min(windowSize, Math.max(layer.seqLen, seen));
|
|
246
278
|
}
|
|
247
279
|
|
|
280
|
+
async _recordRingF32ToF16(recorder, layer, spec, keysBuffer, valuesBuffer, startPos, numTokens) {
|
|
281
|
+
const windowSize = spec.capacityTokens;
|
|
282
|
+
const fullStart = startPos;
|
|
283
|
+
const fullTokens = numTokens;
|
|
284
|
+
let srcElementOffset = 0;
|
|
285
|
+
|
|
286
|
+
if (numTokens > windowSize) {
|
|
287
|
+
const dropTokens = numTokens - windowSize;
|
|
288
|
+
startPos += dropTokens;
|
|
289
|
+
numTokens = windowSize;
|
|
290
|
+
srcElementOffset = dropTokens * spec.kvSize;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
const sourceBytesNeeded = (srcElementOffset + (numTokens * spec.kvSize)) * 4;
|
|
294
|
+
if (sourceBytesNeeded > keysBuffer.size || sourceBytesNeeded > valuesBuffer.size) {
|
|
295
|
+
throw new Error('MixedGeometryKVCache ring f32-to-f16 write buffer is smaller than the requested write.');
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
const writePos = startPos % windowSize;
|
|
299
|
+
const firstChunkTokens = Math.min(numTokens, windowSize - writePos);
|
|
300
|
+
const secondChunkTokens = numTokens - firstChunkTokens;
|
|
301
|
+
|
|
302
|
+
await recordKVCacheWriteF32ToF16(
|
|
303
|
+
recorder,
|
|
304
|
+
keysBuffer,
|
|
305
|
+
valuesBuffer,
|
|
306
|
+
layer.keysGPU,
|
|
307
|
+
layer.valuesGPU,
|
|
308
|
+
{
|
|
309
|
+
srcOffset: srcElementOffset,
|
|
310
|
+
dstOffset: writePos * spec.kvSize,
|
|
311
|
+
elementCount: firstChunkTokens * spec.kvSize,
|
|
312
|
+
}
|
|
313
|
+
);
|
|
314
|
+
|
|
315
|
+
if (secondChunkTokens > 0) {
|
|
316
|
+
await recordKVCacheWriteF32ToF16(
|
|
317
|
+
recorder,
|
|
318
|
+
keysBuffer,
|
|
319
|
+
valuesBuffer,
|
|
320
|
+
layer.keysGPU,
|
|
321
|
+
layer.valuesGPU,
|
|
322
|
+
{
|
|
323
|
+
srcOffset: srcElementOffset + (firstChunkTokens * spec.kvSize),
|
|
324
|
+
dstOffset: 0,
|
|
325
|
+
elementCount: secondChunkTokens * spec.kvSize,
|
|
326
|
+
}
|
|
327
|
+
);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
const seen = Math.max(this.totalTokensSeen, fullStart + fullTokens);
|
|
331
|
+
layer.seqLen = Math.min(windowSize, Math.max(layer.seqLen, seen));
|
|
332
|
+
}
|
|
333
|
+
|
|
248
334
|
_updateMetadata(layer, spec, startPos, numTokens) {
|
|
249
335
|
if (spec.layout !== 'ring') {
|
|
250
336
|
layer.seqLen = Math.max(layer.seqLen, Math.min(startPos + numTokens, spec.capacityTokens));
|
|
@@ -304,6 +390,27 @@ export class MixedGeometryKVCache {
|
|
|
304
390
|
this._updateMetadata(layer, spec, startPos, numTokens);
|
|
305
391
|
}
|
|
306
392
|
|
|
393
|
+
async recordUpdateF32ToF16FromGPU(recorder, layerIdx, keysBuffer, valuesBuffer, startPos, numTokens) {
|
|
394
|
+
this._assertLayerIndex(layerIdx);
|
|
395
|
+
this._assertStartPos(startPos);
|
|
396
|
+
if (this.kvDtype !== 'f16') {
|
|
397
|
+
throw new Error('MixedGeometryKVCache recordUpdateF32ToF16FromGPU requires an f16 KV cache.');
|
|
398
|
+
}
|
|
399
|
+
this._assertTokenCount(numTokens, 'MixedGeometryKVCache recordUpdateF32ToF16FromGPU');
|
|
400
|
+
if (numTokens === 0) {
|
|
401
|
+
return;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
const spec = this.layerSpecs[layerIdx];
|
|
405
|
+
const layer = this.layers[layerIdx];
|
|
406
|
+
if (spec.layout === 'ring') {
|
|
407
|
+
await this._recordRingF32ToF16(recorder, layer, spec, keysBuffer, valuesBuffer, startPos, numTokens);
|
|
408
|
+
} else {
|
|
409
|
+
await this._recordContiguousF32ToF16(recorder, layer, spec, keysBuffer, valuesBuffer, startPos, numTokens);
|
|
410
|
+
}
|
|
411
|
+
this._updateMetadata(layer, spec, startPos, numTokens);
|
|
412
|
+
}
|
|
413
|
+
|
|
307
414
|
get() {
|
|
308
415
|
throw new Error('MixedGeometryKVCache CPU get() is not implemented.');
|
|
309
416
|
}
|
|
@@ -68,5 +68,19 @@ export class SlidingWindowKVCache extends KVCache {
|
|
|
68
68
|
tokenIds?: number[] | null
|
|
69
69
|
): void;
|
|
70
70
|
|
|
71
|
+
/**
|
|
72
|
+
* Record direct f32 source to f16 GPU KV cache update with ring-buffer semantics.
|
|
73
|
+
* Does NOT submit - caller is responsible for submitting the recorder.
|
|
74
|
+
*/
|
|
75
|
+
recordUpdateF32ToF16FromGPU(
|
|
76
|
+
recorder: import('../../gpu/kernel-selector.js').CommandRecorder,
|
|
77
|
+
layerIdx: number,
|
|
78
|
+
keysBuffer: GPUBuffer,
|
|
79
|
+
valuesBuffer: GPUBuffer,
|
|
80
|
+
startPos: number,
|
|
81
|
+
numTokens: number,
|
|
82
|
+
tokenIds?: number[] | null
|
|
83
|
+
): void | Promise<void>;
|
|
84
|
+
|
|
71
85
|
getMemoryStats(): MemoryStats & { windowSize: number; totalTokensSeen: number };
|
|
72
86
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
3
|
import { getDevice } from '../../gpu/device.js';
|
|
4
|
+
import { recordKVCacheWriteF32ToF16 } from '../../gpu/kernel-selector.js';
|
|
4
5
|
import { KVCache } from './base.js';
|
|
5
6
|
|
|
6
7
|
// ============================================================================
|
|
@@ -200,6 +201,93 @@ export class SlidingWindowKVCache extends KVCache {
|
|
|
200
201
|
}
|
|
201
202
|
}
|
|
202
203
|
|
|
204
|
+
|
|
205
|
+
async recordUpdateF32ToF16FromGPU(
|
|
206
|
+
recorder,
|
|
207
|
+
layerIdx,
|
|
208
|
+
keysBuffer,
|
|
209
|
+
valuesBuffer,
|
|
210
|
+
startPos,
|
|
211
|
+
numTokens
|
|
212
|
+
) {
|
|
213
|
+
this._assertLayerIndex(layerIdx);
|
|
214
|
+
this._assertStartPos(startPos);
|
|
215
|
+
if (this.kvDtype !== 'f16') {
|
|
216
|
+
throw new Error('SlidingWindowKVCache recordUpdateF32ToF16FromGPU requires an f16 KV cache.');
|
|
217
|
+
}
|
|
218
|
+
if (!Number.isInteger(numTokens) || numTokens < 0) {
|
|
219
|
+
throw new Error('SlidingWindowKVCache recordUpdateF32ToF16FromGPU requires a non-negative integer token count.');
|
|
220
|
+
}
|
|
221
|
+
if (numTokens === 0) {
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const layer = (this.layers[layerIdx]);
|
|
226
|
+
if (!layer.keysGPU) {
|
|
227
|
+
throw new Error('GPU cache not initialized');
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
const windowSize = this.windowSize;
|
|
231
|
+
const sourceBytesPerToken = this.kvSize * 4;
|
|
232
|
+
const fullStart = startPos;
|
|
233
|
+
const fullTokens = numTokens;
|
|
234
|
+
let srcElementOffset = 0;
|
|
235
|
+
|
|
236
|
+
if (numTokens > windowSize) {
|
|
237
|
+
const dropTokens = numTokens - windowSize;
|
|
238
|
+
startPos += dropTokens;
|
|
239
|
+
numTokens = windowSize;
|
|
240
|
+
srcElementOffset = dropTokens * this.kvSize;
|
|
241
|
+
}
|
|
242
|
+
const writePos = startPos % windowSize;
|
|
243
|
+
const sourceBytesNeeded = (srcElementOffset * 4) + (numTokens * sourceBytesPerToken);
|
|
244
|
+
if (sourceBytesNeeded > keysBuffer.size || sourceBytesNeeded > valuesBuffer.size) {
|
|
245
|
+
throw new Error('SlidingWindowKVCache recordUpdateF32ToF16FromGPU buffer size is smaller than requested write.');
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
const firstChunkTokens = Math.min(numTokens, windowSize - writePos);
|
|
249
|
+
const secondChunkTokens = numTokens - firstChunkTokens;
|
|
250
|
+
|
|
251
|
+
await recordKVCacheWriteF32ToF16(
|
|
252
|
+
recorder,
|
|
253
|
+
keysBuffer,
|
|
254
|
+
valuesBuffer,
|
|
255
|
+
layer.keysGPU,
|
|
256
|
+
layer.valuesGPU,
|
|
257
|
+
{
|
|
258
|
+
srcOffset: srcElementOffset,
|
|
259
|
+
dstOffset: writePos * this.kvSize,
|
|
260
|
+
elementCount: firstChunkTokens * this.kvSize,
|
|
261
|
+
}
|
|
262
|
+
);
|
|
263
|
+
|
|
264
|
+
if (secondChunkTokens > 0) {
|
|
265
|
+
await recordKVCacheWriteF32ToF16(
|
|
266
|
+
recorder,
|
|
267
|
+
keysBuffer,
|
|
268
|
+
valuesBuffer,
|
|
269
|
+
layer.keysGPU,
|
|
270
|
+
layer.valuesGPU,
|
|
271
|
+
{
|
|
272
|
+
srcOffset: srcElementOffset + (firstChunkTokens * this.kvSize),
|
|
273
|
+
dstOffset: 0,
|
|
274
|
+
elementCount: secondChunkTokens * this.kvSize,
|
|
275
|
+
}
|
|
276
|
+
);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
const seen = Math.max(this.totalTokensSeen, fullStart + fullTokens);
|
|
280
|
+
const storedLen = Math.min(windowSize, Math.max(0, seen));
|
|
281
|
+
this.totalTokensSeen = Math.max(this.totalTokensSeen, seen);
|
|
282
|
+
const prevLayerSeqLen = Math.min(Math.max(layer.seqLen || 0, 0), windowSize);
|
|
283
|
+
|
|
284
|
+
layer.seqLen = Math.max(prevLayerSeqLen, storedLen);
|
|
285
|
+
if (layerIdx === this.numLayers - 1) {
|
|
286
|
+
this.currentSeqLen = Math.min(this.currentSeqLen, windowSize);
|
|
287
|
+
this.currentSeqLen = Math.max(this.currentSeqLen, storedLen);
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
203
291
|
|
|
204
292
|
_slideWindow(numNewTokens) {
|
|
205
293
|
if (!Number.isInteger(numNewTokens) || numNewTokens <= 0) {
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { CatalogModelLane } from '../../config/model-lanes/catalog-lane-resolver.js';
|
|
2
|
+
|
|
3
|
+
export interface KernelCapabilities {
|
|
4
|
+
hasF16?: boolean;
|
|
5
|
+
hasSubgroups?: boolean;
|
|
6
|
+
hasSubgroupsF16?: boolean;
|
|
7
|
+
maxBufferSize?: number;
|
|
8
|
+
maxWorkgroupSize?: number;
|
|
9
|
+
maxWorkgroupStorageSize?: number;
|
|
10
|
+
adapterInfo?: {
|
|
11
|
+
vendor?: string;
|
|
12
|
+
architecture?: string;
|
|
13
|
+
device?: string;
|
|
14
|
+
description?: string;
|
|
15
|
+
};
|
|
16
|
+
[key: string]: unknown;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface ExecutionLaneSelection {
|
|
20
|
+
entry: CatalogModelLane;
|
|
21
|
+
selectedModelId: string | null;
|
|
22
|
+
usedFallback: boolean;
|
|
23
|
+
rejected: Array<{
|
|
24
|
+
modelId: string | null;
|
|
25
|
+
reason: string;
|
|
26
|
+
}>;
|
|
27
|
+
reason: string;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export declare function hasF16SubgroupLaneSupport(capabilities: KernelCapabilities | null | undefined): boolean;
|
|
31
|
+
|
|
32
|
+
export declare function assertExecutionLaneManifestSupported(
|
|
33
|
+
entry: CatalogModelLane | null | undefined,
|
|
34
|
+
manifest: Record<string, unknown>,
|
|
35
|
+
capabilities: KernelCapabilities | null | undefined,
|
|
36
|
+
options?: {
|
|
37
|
+
normalizeManifest?: (manifest: Record<string, unknown>) => Record<string, unknown>;
|
|
38
|
+
kernelPathPolicy?: unknown;
|
|
39
|
+
platform?: {
|
|
40
|
+
id?: string;
|
|
41
|
+
vendor?: string;
|
|
42
|
+
architecture?: string;
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
): void;
|
|
46
|
+
|
|
47
|
+
export declare function selectExecutionLaneForCapabilities(
|
|
48
|
+
entry: CatalogModelLane,
|
|
49
|
+
options?: {
|
|
50
|
+
capabilities?: KernelCapabilities | null;
|
|
51
|
+
manifestByModelId?: Map<string, Record<string, unknown>> | Record<string, Record<string, unknown>>;
|
|
52
|
+
manifests?: Map<string, Record<string, unknown>> | Record<string, Record<string, unknown>>;
|
|
53
|
+
normalizeManifest?: (manifest: Record<string, unknown>) => Record<string, unknown>;
|
|
54
|
+
kernelPathPolicy?: unknown;
|
|
55
|
+
platform?: {
|
|
56
|
+
id?: string;
|
|
57
|
+
vendor?: string;
|
|
58
|
+
architecture?: string;
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
): ExecutionLaneSelection;
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { DEFAULT_RUNTIME_CONFIG } from '../../config/schema/index.js';
|
|
2
|
+
import { compileExecutionV1 } from '../pipelines/text/execution-v1.js';
|
|
3
|
+
|
|
4
|
+
export function hasF16SubgroupLaneSupport(capabilities) {
|
|
5
|
+
return capabilities?.hasF16 === true && capabilities?.hasSubgroups === true;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
function buildPlatform(capabilities) {
|
|
9
|
+
return {
|
|
10
|
+
id: 'model-lane-runtime',
|
|
11
|
+
vendor: capabilities?.adapterInfo?.vendor ?? 'unknown',
|
|
12
|
+
architecture: capabilities?.adapterInfo?.architecture ?? 'unknown',
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function getManifest(manifestByModelId, modelId) {
|
|
17
|
+
if (!manifestByModelId || !modelId) return null;
|
|
18
|
+
if (typeof manifestByModelId.get === 'function') return manifestByModelId.get(modelId) ?? null;
|
|
19
|
+
return manifestByModelId[modelId] ?? null;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function cloneManifest(manifest) {
|
|
23
|
+
return typeof structuredClone === 'function'
|
|
24
|
+
? structuredClone(manifest)
|
|
25
|
+
: JSON.parse(JSON.stringify(manifest));
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function assertExecutionLaneManifestSupported(entry, manifest, capabilities, options = {}) {
|
|
29
|
+
const normalizedManifest = typeof options.normalizeManifest === 'function'
|
|
30
|
+
? options.normalizeManifest(cloneManifest(manifest))
|
|
31
|
+
: cloneManifest(manifest);
|
|
32
|
+
compileExecutionV1({
|
|
33
|
+
manifestInference: normalizedManifest.inference,
|
|
34
|
+
modelId: normalizedManifest.modelId ?? entry?.modelId ?? 'unknown',
|
|
35
|
+
numLayers: normalizedManifest.architecture?.numLayers ?? 0,
|
|
36
|
+
headDim: normalizedManifest.architecture?.headDim ?? null,
|
|
37
|
+
capabilities,
|
|
38
|
+
platform: options.platform ?? buildPlatform(capabilities),
|
|
39
|
+
kernelPathPolicy: options.kernelPathPolicy ?? DEFAULT_RUNTIME_CONFIG.inference.kernelPathPolicy,
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function unsupportedError(entry, fallback, preferredError, fallbackError) {
|
|
44
|
+
return new Error(
|
|
45
|
+
`${entry?.modelId ?? 'unknown'}: preferred lane is unsupported on this device (${preferredError.message}); ` +
|
|
46
|
+
`${fallback?.modelId ?? 'unknown'}: fallback lane is also unsupported (${fallbackError.message}).`
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function selectExecutionLaneForCapabilities(entry, options = {}) {
|
|
51
|
+
const capabilities = options.capabilities;
|
|
52
|
+
const manifestByModelId = options.manifestByModelId ?? options.manifests;
|
|
53
|
+
const fallback = entry?.demoFallbackVariant ?? null;
|
|
54
|
+
const manifest = getManifest(manifestByModelId, entry?.modelId);
|
|
55
|
+
if (!manifest) {
|
|
56
|
+
throw new Error(`${entry?.modelId ?? 'unknown'}: execution lane preflight missing manifest.`);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const assertSupported = (candidate, candidateManifest) => assertExecutionLaneManifestSupported(
|
|
60
|
+
candidate,
|
|
61
|
+
candidateManifest,
|
|
62
|
+
capabilities,
|
|
63
|
+
{
|
|
64
|
+
normalizeManifest: options.normalizeManifest,
|
|
65
|
+
kernelPathPolicy: options.kernelPathPolicy,
|
|
66
|
+
platform: options.platform,
|
|
67
|
+
}
|
|
68
|
+
);
|
|
69
|
+
|
|
70
|
+
if (!fallback) {
|
|
71
|
+
assertSupported(entry, manifest);
|
|
72
|
+
return {
|
|
73
|
+
entry,
|
|
74
|
+
selectedModelId: entry?.modelId ?? null,
|
|
75
|
+
usedFallback: false,
|
|
76
|
+
rejected: [],
|
|
77
|
+
reason: 'primary lane supported',
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const fallbackManifest = getManifest(manifestByModelId, fallback.modelId);
|
|
82
|
+
if (!fallbackManifest) {
|
|
83
|
+
throw new Error(`${fallback.modelId ?? 'unknown'}: execution lane fallback preflight missing manifest.`);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
if (!hasF16SubgroupLaneSupport(capabilities)) {
|
|
87
|
+
assertSupported(fallback, fallbackManifest);
|
|
88
|
+
return {
|
|
89
|
+
entry: fallback,
|
|
90
|
+
selectedModelId: fallback.modelId ?? null,
|
|
91
|
+
usedFallback: true,
|
|
92
|
+
rejected: [{
|
|
93
|
+
modelId: entry?.modelId ?? null,
|
|
94
|
+
reason: 'device lacks shader-f16 plus subgroups',
|
|
95
|
+
}],
|
|
96
|
+
reason: 'fallback selected because preferred lane requires shader-f16 plus subgroups',
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
try {
|
|
101
|
+
assertSupported(entry, manifest);
|
|
102
|
+
return {
|
|
103
|
+
entry,
|
|
104
|
+
selectedModelId: entry?.modelId ?? null,
|
|
105
|
+
usedFallback: false,
|
|
106
|
+
rejected: [],
|
|
107
|
+
reason: 'preferred lane supported',
|
|
108
|
+
};
|
|
109
|
+
} catch (preferredError) {
|
|
110
|
+
try {
|
|
111
|
+
assertSupported(fallback, fallbackManifest);
|
|
112
|
+
return {
|
|
113
|
+
entry: fallback,
|
|
114
|
+
selectedModelId: fallback.modelId ?? null,
|
|
115
|
+
usedFallback: true,
|
|
116
|
+
rejected: [{
|
|
117
|
+
modelId: entry?.modelId ?? null,
|
|
118
|
+
reason: preferredError.message,
|
|
119
|
+
}],
|
|
120
|
+
reason: 'fallback selected after preferred lane preflight rejection',
|
|
121
|
+
};
|
|
122
|
+
} catch (fallbackError) {
|
|
123
|
+
throw unsupportedError(entry, fallback, preferredError, fallbackError);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
@@ -53,6 +53,8 @@ interface LoadBalanceStats {
|
|
|
53
53
|
totalTokens: number;
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
+
type RouterVector = Float32Array | GPUBuffer | import('../gpu/weight-buffer.js').WeightBuffer;
|
|
57
|
+
|
|
56
58
|
/**
|
|
57
59
|
* Expert utilization stats
|
|
58
60
|
*/
|
|
@@ -77,6 +79,10 @@ export declare class MoERouter {
|
|
|
77
79
|
gateWeight: Float32Array | GPUBuffer | import('../gpu/weight-buffer.js').WeightBuffer | null;
|
|
78
80
|
// Router bias (optional, used by GPT-OSS)
|
|
79
81
|
gateBias: Float32Array | GPUBuffer | null;
|
|
82
|
+
// Optional DiffusionGemma router input scale
|
|
83
|
+
gateScale: RouterVector | null;
|
|
84
|
+
// Optional DiffusionGemma per-expert output scale
|
|
85
|
+
perExpertScale: RouterVector | null;
|
|
80
86
|
|
|
81
87
|
// Track active experts for the current batch
|
|
82
88
|
activeExperts: Set<number>;
|
|
@@ -91,8 +97,15 @@ export declare class MoERouter {
|
|
|
91
97
|
* Load router gate weights from model
|
|
92
98
|
* @param weights - Gate weight matrix [hidden_size, num_experts]
|
|
93
99
|
* @param bias - Optional gate bias vector [num_experts]
|
|
100
|
+
* @param scale - Optional router input scale vector [hidden_size]
|
|
101
|
+
* @param perExpertScale - Optional selected-expert weight scale [num_experts]
|
|
94
102
|
*/
|
|
95
|
-
loadWeights(
|
|
103
|
+
loadWeights(
|
|
104
|
+
weights: Float32Array | GPUBuffer | import('../gpu/weight-buffer.js').WeightBuffer,
|
|
105
|
+
bias?: Float32Array | GPUBuffer | null,
|
|
106
|
+
scale?: RouterVector | null,
|
|
107
|
+
perExpertScale?: RouterVector | null
|
|
108
|
+
): void;
|
|
96
109
|
|
|
97
110
|
/**
|
|
98
111
|
* Compute router logits from hidden states (CPU fallback)
|