doppler-gpu 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +67 -34
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.js +5 -1
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/version.js +1 -1
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { Q4K_BLOCK_BYTES, QK_K } from '../../config/schema/quantization.schema.js';
|
|
2
2
|
|
|
3
|
-
const SUPPORTED_PACKINGS = new Set(['dense', 'q4k', 'q4_0', 'gguf-block-v2']);
|
|
3
|
+
const SUPPORTED_PACKINGS = new Set(['dense', 'q4k', 'q4_0', 'w4a16', 'gguf-block-v2']);
|
|
4
4
|
const Q4_0_BLOCK_BYTES = 18;
|
|
5
5
|
const Q4_0_BLOCK_VALUES = 32;
|
|
6
|
+
const W4A16_BLOCK_BYTES = 16;
|
|
7
|
+
const W4A16_BLOCK_VALUES = 32;
|
|
6
8
|
|
|
7
9
|
function fail(label, message) {
|
|
8
10
|
throw new Error(`[RDRRStorage] ${label}: ${message}`);
|
|
@@ -178,9 +180,14 @@ export function getPackingByteLayout(storage, tensor) {
|
|
|
178
180
|
blockBytes: descriptor.blockBytes ?? Q4_0_BLOCK_BYTES,
|
|
179
181
|
};
|
|
180
182
|
}
|
|
183
|
+
if (descriptor.packing === 'w4a16') {
|
|
184
|
+
return {
|
|
185
|
+
blockElementCount: blockElementCount || W4A16_BLOCK_VALUES,
|
|
186
|
+
blockBytes: descriptor.blockBytes ?? W4A16_BLOCK_BYTES,
|
|
187
|
+
};
|
|
188
|
+
}
|
|
181
189
|
return {
|
|
182
190
|
blockElementCount,
|
|
183
191
|
blockBytes: descriptor.blockBytes,
|
|
184
192
|
};
|
|
185
193
|
}
|
|
186
|
-
|
|
@@ -22,6 +22,7 @@ import type {
|
|
|
22
22
|
ManifestInferenceSchema,
|
|
23
23
|
type TensorRole as SchemaTensorRole,
|
|
24
24
|
} from '../../config/schema/index.js';
|
|
25
|
+
import type { FunctionalDescriptorManifest } from './functional-descriptor.js';
|
|
25
26
|
|
|
26
27
|
// =============================================================================
|
|
27
28
|
// Re-exports from Schema
|
|
@@ -91,7 +92,7 @@ export interface TensorStorageShardSpan {
|
|
|
91
92
|
}
|
|
92
93
|
|
|
93
94
|
export interface TensorPhysicalStorageDescriptor {
|
|
94
|
-
packing: 'dense' | 'q4k' | 'q4_0' | 'gguf-block-v2';
|
|
95
|
+
packing: 'dense' | 'q4k' | 'q4_0' | 'w4a16' | 'gguf-block-v2';
|
|
95
96
|
blockShape?: number[];
|
|
96
97
|
blockBytes?: number;
|
|
97
98
|
companions?: TensorStorageCompanion[];
|
|
@@ -133,9 +134,9 @@ export interface LayerConfig {
|
|
|
133
134
|
export interface ComponentGroup extends ComponentGroupSchema {}
|
|
134
135
|
|
|
135
136
|
export interface TensorLocation {
|
|
136
|
-
shard
|
|
137
|
+
shard?: number;
|
|
137
138
|
shardIndex?: number;
|
|
138
|
-
offset
|
|
139
|
+
offset?: number;
|
|
139
140
|
size: number;
|
|
140
141
|
shape: number[];
|
|
141
142
|
dtype: string;
|
|
@@ -146,6 +147,7 @@ export interface TensorLocation {
|
|
|
146
147
|
originalShape?: number[];
|
|
147
148
|
sourceTransform?: TensorSourceTransform;
|
|
148
149
|
storage?: TensorPhysicalStorageDescriptor;
|
|
150
|
+
descriptorManifest?: FunctionalDescriptorManifest;
|
|
149
151
|
}
|
|
150
152
|
|
|
151
153
|
export interface ConversionInfo {
|
|
@@ -2,6 +2,11 @@ import { validateTensorConfigConsistency } from './tensor-config-validator.js';
|
|
|
2
2
|
import { validateManifestExecutionContract } from '../../config/execution-contract-check.js';
|
|
3
3
|
import { validateManifestInference } from '../../config/schema/index.js';
|
|
4
4
|
import { validateTensorStorageDescriptor } from './storage-descriptor.js';
|
|
5
|
+
import {
|
|
6
|
+
getFunctionalDescriptorManifest,
|
|
7
|
+
isFunctionalDescriptorDtype,
|
|
8
|
+
validateFunctionalDescriptorManifest,
|
|
9
|
+
} from './functional-descriptor.js';
|
|
5
10
|
|
|
6
11
|
const ARTIFACT_COMPLETENESS_VALUES = new Set([
|
|
7
12
|
'complete',
|
|
@@ -371,6 +376,15 @@ export function validateManifest(manifest) {
|
|
|
371
376
|
if (!tensor.role || typeof tensor.role !== 'string') {
|
|
372
377
|
errors.push(`Tensor "${name}" missing role`);
|
|
373
378
|
}
|
|
379
|
+
if (isFunctionalDescriptorDtype(tensor.dtype)) {
|
|
380
|
+
const descriptorValidation = validateFunctionalDescriptorManifest(
|
|
381
|
+
getFunctionalDescriptorManifest(tensor),
|
|
382
|
+
`Tensor "${name}" descriptorManifest`
|
|
383
|
+
);
|
|
384
|
+
for (const descriptorError of descriptorValidation.errors) {
|
|
385
|
+
errors.push(descriptorError);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
374
388
|
if (tensor.storage !== undefined) {
|
|
375
389
|
validateTensorStorageDescriptor(tensor.storage, `tensor "${name}"`, errors);
|
|
376
390
|
}
|
|
@@ -388,7 +402,7 @@ export function validateManifest(manifest) {
|
|
|
388
402
|
}
|
|
389
403
|
if (typeof moe.expertFormat !== 'string') {
|
|
390
404
|
errors.push('Invalid moeConfig.expertFormat');
|
|
391
|
-
} else if (moe.expertFormat !== 'mixtral' && moe.expertFormat !== 'gpt-oss') {
|
|
405
|
+
} else if (moe.expertFormat !== 'mixtral' && moe.expertFormat !== 'gpt-oss' && moe.expertFormat !== 'gemma4') {
|
|
392
406
|
errors.push(`Invalid moeConfig.expertFormat: ${moe.expertFormat}`);
|
|
393
407
|
}
|
|
394
408
|
if (moe.numExpertsPerToken > moe.numExperts) {
|
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
export interface RecorderOptions {
|
|
10
10
|
/** Enable GPU timestamp profiling (requires 'timestamp-query' feature) */
|
|
11
11
|
profile?: boolean;
|
|
12
|
+
/** Collect per-label operation counts for diagnostics. Defaults to true. */
|
|
13
|
+
recordLabels?: boolean;
|
|
12
14
|
}
|
|
13
15
|
|
|
14
16
|
/** Profiling result - maps kernel label to time in milliseconds */
|
|
@@ -81,6 +83,33 @@ export declare class CommandRecorder {
|
|
|
81
83
|
*/
|
|
82
84
|
beginComputePass(label?: string): GPUComputePassEncoder;
|
|
83
85
|
|
|
86
|
+
/**
|
|
87
|
+
* Close the active coalesced compute pass, if one is open.
|
|
88
|
+
*/
|
|
89
|
+
closeActiveComputePass(): void;
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Record a simple dispatch. Non-profiling recorders coalesce consecutive
|
|
93
|
+
* dispatches into one compute pass until a raw encoder boundary.
|
|
94
|
+
*/
|
|
95
|
+
recordDispatch(
|
|
96
|
+
pipeline: GPUComputePipeline,
|
|
97
|
+
bindGroup: GPUBindGroup,
|
|
98
|
+
workgroups: [number, number, number],
|
|
99
|
+
label?: string
|
|
100
|
+
): void;
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Record an indirect dispatch with the same pass coalescing behavior.
|
|
104
|
+
*/
|
|
105
|
+
recordDispatchIndirect(
|
|
106
|
+
pipeline: GPUComputePipeline,
|
|
107
|
+
bindGroup: GPUBindGroup,
|
|
108
|
+
indirectBuffer: GPUBuffer,
|
|
109
|
+
indirectOffset?: number,
|
|
110
|
+
label?: string
|
|
111
|
+
): void;
|
|
112
|
+
|
|
84
113
|
/**
|
|
85
114
|
* Get the raw encoder for advanced use cases.
|
|
86
115
|
* @returns GPUCommandEncoder
|
|
@@ -125,7 +154,14 @@ export declare class CommandRecorder {
|
|
|
125
154
|
* Get statistics about recorded operations.
|
|
126
155
|
* @returns Statistics object
|
|
127
156
|
*/
|
|
128
|
-
getStats(): {
|
|
157
|
+
getStats(): {
|
|
158
|
+
opCount: number;
|
|
159
|
+
opLabelCounts: Record<string, number>;
|
|
160
|
+
computePassCount: number;
|
|
161
|
+
tempBufferCount: number;
|
|
162
|
+
pooledBufferCount: number;
|
|
163
|
+
submitted: boolean;
|
|
164
|
+
};
|
|
129
165
|
|
|
130
166
|
/**
|
|
131
167
|
* Get the submit completion latency in milliseconds (null if not resolved yet).
|
|
@@ -165,7 +201,11 @@ export declare class CommandRecorder {
|
|
|
165
201
|
* @param options - Recorder options
|
|
166
202
|
* @returns CommandRecorder instance
|
|
167
203
|
*/
|
|
168
|
-
export function createCommandRecorder(
|
|
204
|
+
export function createCommandRecorder(
|
|
205
|
+
label?: string,
|
|
206
|
+
options?: RecorderOptions,
|
|
207
|
+
device?: GPUDevice | null
|
|
208
|
+
): CommandRecorder;
|
|
169
209
|
|
|
170
210
|
/**
|
|
171
211
|
* Create a profiling-enabled CommandRecorder.
|
|
@@ -174,6 +214,6 @@ export function createCommandRecorder(label?: string, options?: RecorderOptions)
|
|
|
174
214
|
* @param label - Label for debugging
|
|
175
215
|
* @returns CommandRecorder with profiling enabled
|
|
176
216
|
*/
|
|
177
|
-
export function createProfilingRecorder(label?: string): CommandRecorder;
|
|
217
|
+
export function createProfilingRecorder(label?: string, device?: GPUDevice | null): CommandRecorder;
|
|
178
218
|
|
|
179
219
|
export default CommandRecorder;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import { getDevice, hasFeature, FEATURES } from './device.js';
|
|
4
4
|
import { allowReadback, trackAllocation } from './perf-guards.js';
|
|
5
|
-
import { getUniformCache } from './uniform-cache.js';
|
|
5
|
+
import { getUniformCache, toUniformArrayBuffer } from './uniform-cache.js';
|
|
6
6
|
import { isBufferActive, isPersistentBuffer, releaseBuffer, discardBuffer } from '../memory/buffer-pool.js';
|
|
7
7
|
import { log } from '../debug/index.js';
|
|
8
8
|
import { getRuntimeConfig } from '../config/runtime.js';
|
|
@@ -11,8 +11,6 @@ let didLogQueryClamp = false;
|
|
|
11
11
|
let didLogQueryFallback = false;
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
|
|
16
14
|
export class CommandRecorder {
|
|
17
15
|
|
|
18
16
|
device;
|
|
@@ -38,12 +36,19 @@ export class CommandRecorder {
|
|
|
38
36
|
|
|
39
37
|
#completionTasks;
|
|
40
38
|
|
|
41
|
-
|
|
42
39
|
#submitted;
|
|
43
40
|
|
|
44
41
|
|
|
45
42
|
#opCount;
|
|
46
43
|
|
|
44
|
+
#opLabelCounts;
|
|
45
|
+
|
|
46
|
+
#recordLabels;
|
|
47
|
+
|
|
48
|
+
#computePassCount;
|
|
49
|
+
|
|
50
|
+
#activeComputePass = null;
|
|
51
|
+
|
|
47
52
|
// Profiling state
|
|
48
53
|
|
|
49
54
|
#profilingEnabled;
|
|
@@ -89,7 +94,10 @@ export class CommandRecorder {
|
|
|
89
94
|
|
|
90
95
|
// Operation count for debugging
|
|
91
96
|
this.#opCount = 0;
|
|
92
|
-
|
|
97
|
+
this.#recordLabels = options.recordLabels !== false;
|
|
98
|
+
this.#opLabelCounts = this.#recordLabels ? Object.create(null) : null;
|
|
99
|
+
this.#computePassCount = 0;
|
|
100
|
+
this.#activeComputePass = null;
|
|
93
101
|
// Initialize profiling if requested and available
|
|
94
102
|
this.#profilingEnabled = options.profile === true && hasFeature(FEATURES.TIMESTAMP_QUERY);
|
|
95
103
|
if (this.#profilingEnabled) {
|
|
@@ -218,34 +226,48 @@ export class CommandRecorder {
|
|
|
218
226
|
|
|
219
227
|
|
|
220
228
|
createUniformBuffer(data, label = 'uniforms') {
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
? data
|
|
224
|
-
: data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
|
|
229
|
+
return getUniformCache().getOrCreate(toUniformArrayBuffer(data), label);
|
|
230
|
+
}
|
|
225
231
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
232
|
+
#normalizeOperationLabel(label) {
|
|
233
|
+
return typeof label === 'string' && label.length > 0
|
|
234
|
+
? label
|
|
235
|
+
: 'compute_pass';
|
|
229
236
|
}
|
|
230
237
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
238
|
+
#recordOperation(label) {
|
|
239
|
+
const opLabel = this.#normalizeOperationLabel(label);
|
|
240
|
+
this.#opCount++;
|
|
241
|
+
if (this.#recordLabels) {
|
|
242
|
+
this.#opLabelCounts[opLabel] = (this.#opLabelCounts[opLabel] ?? 0) + 1;
|
|
235
243
|
}
|
|
244
|
+
return opLabel;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
#recordDispatchOperation(label) {
|
|
236
248
|
this.#opCount++;
|
|
249
|
+
if (!this.#recordLabels && !this.#profilingEnabled) {
|
|
250
|
+
return null;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const opLabel = this.#normalizeOperationLabel(label);
|
|
254
|
+
if (this.#recordLabels) {
|
|
255
|
+
this.#opLabelCounts[opLabel] = (this.#opLabelCounts[opLabel] ?? 0) + 1;
|
|
256
|
+
}
|
|
257
|
+
return opLabel;
|
|
258
|
+
}
|
|
237
259
|
|
|
238
|
-
|
|
260
|
+
#beginRawComputePass(opLabel) {
|
|
261
|
+
this.#computePassCount++;
|
|
262
|
+
const passLabel = `${this.label}_${opLabel}_${this.#computePassCount}`;
|
|
239
263
|
|
|
240
|
-
// If profiling enabled, add timestamp writes
|
|
241
264
|
if (this.#profilingEnabled && this.#querySet && this.#nextQueryIndex + 2 <= this.#queryCapacity) {
|
|
242
265
|
const startIndex = this.#nextQueryIndex;
|
|
243
266
|
const endIndex = startIndex + 1;
|
|
244
267
|
this.#nextQueryIndex += 2;
|
|
245
268
|
|
|
246
|
-
// Track this entry for later resolution
|
|
247
269
|
this.#profileEntries.push({
|
|
248
|
-
label,
|
|
270
|
+
label: opLabel,
|
|
249
271
|
startQueryIndex: startIndex,
|
|
250
272
|
endQueryIndex: endIndex,
|
|
251
273
|
});
|
|
@@ -260,17 +282,79 @@ export class CommandRecorder {
|
|
|
260
282
|
});
|
|
261
283
|
}
|
|
262
284
|
|
|
263
|
-
// Non-profiling path
|
|
264
285
|
return this.#encoder.beginComputePass({
|
|
265
286
|
label: passLabel,
|
|
266
287
|
});
|
|
267
288
|
}
|
|
268
289
|
|
|
290
|
+
closeActiveComputePass() {
|
|
291
|
+
if (!this.#activeComputePass) {
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
const pass = this.#activeComputePass;
|
|
295
|
+
this.#activeComputePass = null;
|
|
296
|
+
pass.end();
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
#ensureActiveComputePass() {
|
|
300
|
+
if (!this.#activeComputePass) {
|
|
301
|
+
this.#activeComputePass = this.#beginRawComputePass('coalesced_compute');
|
|
302
|
+
}
|
|
303
|
+
return this.#activeComputePass;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
beginComputePass(label = 'compute_pass') {
|
|
307
|
+
if (this.#submitted) {
|
|
308
|
+
throw new Error('[CommandRecorder] Cannot begin pass after submit');
|
|
309
|
+
}
|
|
310
|
+
this.closeActiveComputePass();
|
|
311
|
+
return this.#beginRawComputePass(this.#recordOperation(label));
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
recordDispatch(pipeline, bindGroup, workgroups, label = 'compute') {
|
|
315
|
+
if (this.#submitted) {
|
|
316
|
+
throw new Error('[CommandRecorder] Cannot record dispatch after submit');
|
|
317
|
+
}
|
|
318
|
+
const opLabel = this.#recordDispatchOperation(label);
|
|
319
|
+
if (this.#profilingEnabled) {
|
|
320
|
+
const pass = this.#beginRawComputePass(opLabel);
|
|
321
|
+
pass.setPipeline(pipeline);
|
|
322
|
+
pass.setBindGroup(0, bindGroup);
|
|
323
|
+
pass.dispatchWorkgroups(workgroups[0], workgroups[1], workgroups[2]);
|
|
324
|
+
pass.end();
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
const pass = this.#ensureActiveComputePass();
|
|
328
|
+
pass.setPipeline(pipeline);
|
|
329
|
+
pass.setBindGroup(0, bindGroup);
|
|
330
|
+
pass.dispatchWorkgroups(workgroups[0], workgroups[1], workgroups[2]);
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
recordDispatchIndirect(pipeline, bindGroup, indirectBuffer, indirectOffset = 0, label = 'compute') {
|
|
334
|
+
if (this.#submitted) {
|
|
335
|
+
throw new Error('[CommandRecorder] Cannot record dispatch after submit');
|
|
336
|
+
}
|
|
337
|
+
const opLabel = this.#recordDispatchOperation(label);
|
|
338
|
+
if (this.#profilingEnabled) {
|
|
339
|
+
const pass = this.#beginRawComputePass(opLabel);
|
|
340
|
+
pass.setPipeline(pipeline);
|
|
341
|
+
pass.setBindGroup(0, bindGroup);
|
|
342
|
+
pass.dispatchWorkgroupsIndirect(indirectBuffer, indirectOffset);
|
|
343
|
+
pass.end();
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
const pass = this.#ensureActiveComputePass();
|
|
347
|
+
pass.setPipeline(pipeline);
|
|
348
|
+
pass.setBindGroup(0, bindGroup);
|
|
349
|
+
pass.dispatchWorkgroupsIndirect(indirectBuffer, indirectOffset);
|
|
350
|
+
}
|
|
351
|
+
|
|
269
352
|
|
|
270
353
|
getEncoder() {
|
|
271
354
|
if (this.#submitted) {
|
|
272
355
|
throw new Error('[CommandRecorder] Cannot access encoder after submit');
|
|
273
356
|
}
|
|
357
|
+
this.closeActiveComputePass();
|
|
274
358
|
return this.#encoder;
|
|
275
359
|
}
|
|
276
360
|
|
|
@@ -365,6 +449,7 @@ export class CommandRecorder {
|
|
|
365
449
|
}
|
|
366
450
|
|
|
367
451
|
const submitStart = performance.now();
|
|
452
|
+
this.closeActiveComputePass();
|
|
368
453
|
const { buffersToDestroy, buffersToRelease } = this.#takeTrackedBuffers();
|
|
369
454
|
try {
|
|
370
455
|
this.device.queue.submit([this.#encoder.finish()]);
|
|
@@ -428,8 +513,18 @@ export class CommandRecorder {
|
|
|
428
513
|
|
|
429
514
|
|
|
430
515
|
getStats() {
|
|
516
|
+
const opLabelCounts = this.#opLabelCounts
|
|
517
|
+
? Object.fromEntries(
|
|
518
|
+
Object.entries(this.#opLabelCounts).sort((a, b) => {
|
|
519
|
+
const countDelta = b[1] - a[1];
|
|
520
|
+
return countDelta !== 0 ? countDelta : a[0].localeCompare(b[0]);
|
|
521
|
+
})
|
|
522
|
+
)
|
|
523
|
+
: {};
|
|
431
524
|
return {
|
|
432
525
|
opCount: this.#opCount,
|
|
526
|
+
opLabelCounts,
|
|
527
|
+
computePassCount: this.#computePassCount,
|
|
433
528
|
tempBufferCount: this.#tempBuffers.length,
|
|
434
529
|
pooledBufferCount: this.#pooledBuffers.length,
|
|
435
530
|
submitted: this.#submitted,
|
|
@@ -444,6 +539,8 @@ export class CommandRecorder {
|
|
|
444
539
|
abort() {
|
|
445
540
|
if (this.#submitted) return;
|
|
446
541
|
|
|
542
|
+
this.closeActiveComputePass();
|
|
543
|
+
|
|
447
544
|
// Destroy temp buffers without submitting
|
|
448
545
|
for (const buffer of this.#tempBuffers) {
|
|
449
546
|
buffer.destroy();
|
|
@@ -562,13 +659,13 @@ export class CommandRecorder {
|
|
|
562
659
|
}
|
|
563
660
|
|
|
564
661
|
|
|
565
|
-
export function createCommandRecorder(label = 'command_recorder', options) {
|
|
566
|
-
return new CommandRecorder(
|
|
662
|
+
export function createCommandRecorder(label = 'command_recorder', options, device = null) {
|
|
663
|
+
return new CommandRecorder(device, label, options);
|
|
567
664
|
}
|
|
568
665
|
|
|
569
666
|
|
|
570
|
-
export function createProfilingRecorder(label = 'profiled_recorder') {
|
|
571
|
-
return new CommandRecorder(
|
|
667
|
+
export function createProfilingRecorder(label = 'profiled_recorder', device = null) {
|
|
668
|
+
return new CommandRecorder(device, label, { profile: true });
|
|
572
669
|
}
|
|
573
670
|
|
|
574
671
|
export default CommandRecorder;
|
package/src/gpu/device.js
CHANGED
|
@@ -9,25 +9,68 @@ import { GB } from '../config/schema/index.js';
|
|
|
9
9
|
// Re-export submit tracker for convenience
|
|
10
10
|
export { setTrackSubmits };
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
const SHARED_DEVICE_STATE_KEY = '__dopplerGpuDeviceState';
|
|
13
|
+
|
|
14
|
+
function getSharedDeviceState() {
|
|
15
|
+
const existing = globalThis[SHARED_DEVICE_STATE_KEY];
|
|
16
|
+
if (existing && typeof existing === 'object') {
|
|
17
|
+
return existing;
|
|
18
|
+
}
|
|
19
|
+
const created = {
|
|
20
|
+
gpuDevice: null,
|
|
21
|
+
kernelCapabilities: null,
|
|
22
|
+
resolvedPlatformConfig: null,
|
|
23
|
+
lastDeviceLossInfo: null,
|
|
24
|
+
platformInitialized: false,
|
|
25
|
+
deviceEpoch: 0,
|
|
26
|
+
};
|
|
27
|
+
Object.defineProperty(globalThis, SHARED_DEVICE_STATE_KEY, {
|
|
28
|
+
value: created,
|
|
29
|
+
writable: false,
|
|
30
|
+
enumerable: false,
|
|
31
|
+
configurable: false,
|
|
32
|
+
});
|
|
33
|
+
return created;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const sharedDeviceState = getSharedDeviceState();
|
|
13
37
|
|
|
14
|
-
let gpuDevice = null;
|
|
38
|
+
let gpuDevice = sharedDeviceState.gpuDevice ?? null;
|
|
15
39
|
|
|
16
|
-
let kernelCapabilities = null;
|
|
40
|
+
let kernelCapabilities = sharedDeviceState.kernelCapabilities ?? null;
|
|
17
41
|
|
|
18
|
-
|
|
42
|
+
let resolvedPlatformConfig = sharedDeviceState.resolvedPlatformConfig ?? null;
|
|
43
|
+
let lastDeviceLossInfo = sharedDeviceState.lastDeviceLossInfo ?? null;
|
|
19
44
|
|
|
20
|
-
let
|
|
21
|
-
let lastDeviceLossInfo = null;
|
|
45
|
+
let platformInitialized = sharedDeviceState.platformInitialized === true;
|
|
22
46
|
|
|
23
|
-
|
|
24
|
-
|
|
47
|
+
let deviceEpoch = Number.isInteger(sharedDeviceState.deviceEpoch)
|
|
48
|
+
? sharedDeviceState.deviceEpoch
|
|
49
|
+
: 0;
|
|
25
50
|
|
|
26
|
-
|
|
27
|
-
|
|
51
|
+
function syncSharedDeviceState() {
|
|
52
|
+
sharedDeviceState.gpuDevice = gpuDevice;
|
|
53
|
+
sharedDeviceState.kernelCapabilities = kernelCapabilities;
|
|
54
|
+
sharedDeviceState.resolvedPlatformConfig = resolvedPlatformConfig;
|
|
55
|
+
sharedDeviceState.lastDeviceLossInfo = lastDeviceLossInfo;
|
|
56
|
+
sharedDeviceState.platformInitialized = platformInitialized;
|
|
57
|
+
sharedDeviceState.deviceEpoch = deviceEpoch;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function hydrateDeviceState() {
|
|
61
|
+
gpuDevice = gpuDevice ?? sharedDeviceState.gpuDevice ?? null;
|
|
62
|
+
kernelCapabilities = kernelCapabilities ?? sharedDeviceState.kernelCapabilities ?? null;
|
|
63
|
+
resolvedPlatformConfig = resolvedPlatformConfig ?? sharedDeviceState.resolvedPlatformConfig ?? null;
|
|
64
|
+
lastDeviceLossInfo = lastDeviceLossInfo ?? sharedDeviceState.lastDeviceLossInfo ?? null;
|
|
65
|
+
platformInitialized = platformInitialized || sharedDeviceState.platformInitialized === true;
|
|
66
|
+
if (Number.isInteger(sharedDeviceState.deviceEpoch) && sharedDeviceState.deviceEpoch > deviceEpoch) {
|
|
67
|
+
deviceEpoch = sharedDeviceState.deviceEpoch;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
28
70
|
|
|
29
71
|
function advanceDeviceEpoch() {
|
|
30
72
|
deviceEpoch += 1;
|
|
73
|
+
syncSharedDeviceState();
|
|
31
74
|
}
|
|
32
75
|
|
|
33
76
|
function clearActiveDeviceState() {
|
|
@@ -35,6 +78,32 @@ function clearActiveDeviceState() {
|
|
|
35
78
|
kernelCapabilities = null;
|
|
36
79
|
resolvedPlatformConfig = null;
|
|
37
80
|
platformInitialized = false;
|
|
81
|
+
syncSharedDeviceState();
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function hasUsableDeviceSlot(device) {
|
|
85
|
+
return isUsableGPUDevice(device);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function buildDeviceStateDiagnostics() {
|
|
89
|
+
const lastLoss = lastDeviceLossInfo
|
|
90
|
+
? {
|
|
91
|
+
reason: lastDeviceLossInfo.reason,
|
|
92
|
+
message: lastDeviceLossInfo.message,
|
|
93
|
+
deviceEpoch: lastDeviceLossInfo.deviceEpoch,
|
|
94
|
+
}
|
|
95
|
+
: null;
|
|
96
|
+
return {
|
|
97
|
+
localDevice: hasUsableDeviceSlot(gpuDevice),
|
|
98
|
+
sharedDevice: hasUsableDeviceSlot(sharedDeviceState.gpuDevice),
|
|
99
|
+
localCapabilities: !!kernelCapabilities,
|
|
100
|
+
sharedCapabilities: !!sharedDeviceState.kernelCapabilities,
|
|
101
|
+
localPlatformInitialized: platformInitialized,
|
|
102
|
+
sharedPlatformInitialized: sharedDeviceState.platformInitialized === true,
|
|
103
|
+
deviceEpoch,
|
|
104
|
+
sharedDeviceEpoch: sharedDeviceState.deviceEpoch,
|
|
105
|
+
lastDeviceLoss: lastLoss,
|
|
106
|
+
};
|
|
38
107
|
}
|
|
39
108
|
|
|
40
109
|
function ensureGpuBufferConstructor(device) {
|
|
@@ -134,10 +203,21 @@ function validateBindGroupDescriptor(descriptor) {
|
|
|
134
203
|
}
|
|
135
204
|
}
|
|
136
205
|
|
|
206
|
+
function shouldValidateBindGroupDescriptors() {
|
|
207
|
+
const env = typeof process !== 'undefined' ? process?.env?.DOPPLER_VALIDATE_BIND_GROUPS : null;
|
|
208
|
+
if (env === '1' || env === 'true') {
|
|
209
|
+
return true;
|
|
210
|
+
}
|
|
211
|
+
return false;
|
|
212
|
+
}
|
|
213
|
+
|
|
137
214
|
function wrapDeviceCreateBindGroup(device) {
|
|
138
215
|
if (!device || device.__dopplerBindGroupValidationWrapped) {
|
|
139
216
|
return device;
|
|
140
217
|
}
|
|
218
|
+
if (!shouldValidateBindGroupDescriptors()) {
|
|
219
|
+
return device;
|
|
220
|
+
}
|
|
141
221
|
const originalCreateBindGroup = device.createBindGroup.bind(device);
|
|
142
222
|
device.createBindGroup = (descriptor) => {
|
|
143
223
|
validateBindGroupDescriptor(descriptor);
|
|
@@ -312,11 +392,13 @@ function buildLimits(adapter) {
|
|
|
312
392
|
|
|
313
393
|
|
|
314
394
|
async function initializePlatformAndRegistry(adapter) {
|
|
395
|
+
hydrateDeviceState();
|
|
315
396
|
if (platformInitialized) {
|
|
316
397
|
return;
|
|
317
398
|
}
|
|
318
399
|
|
|
319
400
|
platformInitialized = true;
|
|
401
|
+
syncSharedDeviceState();
|
|
320
402
|
|
|
321
403
|
try {
|
|
322
404
|
// Dynamic import to avoid circular dependencies and enable hotswap
|
|
@@ -327,6 +409,7 @@ async function initializePlatformAndRegistry(adapter) {
|
|
|
327
409
|
|
|
328
410
|
// Initialize platform detection with the adapter
|
|
329
411
|
resolvedPlatformConfig = await platformLoader.initializePlatform(adapter);
|
|
412
|
+
syncSharedDeviceState();
|
|
330
413
|
|
|
331
414
|
// Preload kernel registry (cached for subsequent calls)
|
|
332
415
|
await kernelRegistry.getRegistry();
|
|
@@ -337,11 +420,13 @@ async function initializePlatformAndRegistry(adapter) {
|
|
|
337
420
|
// Platform/registry init is optional - kernel selection will use fallbacks
|
|
338
421
|
log.warn('GPU', 'Platform/registry init failed (non-fatal): ' + (e).message);
|
|
339
422
|
resolvedPlatformConfig = null;
|
|
423
|
+
syncSharedDeviceState();
|
|
340
424
|
}
|
|
341
425
|
}
|
|
342
426
|
|
|
343
427
|
|
|
344
428
|
export async function initDevice() {
|
|
429
|
+
hydrateDeviceState();
|
|
345
430
|
// Return cached device if available
|
|
346
431
|
if (gpuDevice) {
|
|
347
432
|
if (isUsableGPUDevice(gpuDevice)) {
|
|
@@ -423,6 +508,7 @@ export async function initDevice() {
|
|
|
423
508
|
|
|
424
509
|
const probeMs = await probeSubmitLatency(gpuDevice);
|
|
425
510
|
kernelCapabilities.submitProbeMs = probeMs;
|
|
511
|
+
syncSharedDeviceState();
|
|
426
512
|
|
|
427
513
|
const features = [
|
|
428
514
|
kernelCapabilities.hasF16 && 'f16',
|
|
@@ -435,6 +521,7 @@ export async function initDevice() {
|
|
|
435
521
|
}
|
|
436
522
|
|
|
437
523
|
export function setDevice(device, options = {}) {
|
|
524
|
+
hydrateDeviceState();
|
|
438
525
|
if (!device) {
|
|
439
526
|
clearActiveDeviceState();
|
|
440
527
|
advanceDeviceEpoch();
|
|
@@ -483,31 +570,41 @@ export function setDevice(device, options = {}) {
|
|
|
483
570
|
resolvedPlatformConfig = null;
|
|
484
571
|
platformInitialized = false;
|
|
485
572
|
}
|
|
573
|
+
syncSharedDeviceState();
|
|
486
574
|
}
|
|
487
575
|
|
|
488
576
|
|
|
489
577
|
export function getKernelCapabilities() {
|
|
578
|
+
hydrateDeviceState();
|
|
490
579
|
if (!kernelCapabilities) {
|
|
491
|
-
|
|
580
|
+
const diagnostics = buildDeviceStateDiagnostics();
|
|
581
|
+
throw new Error(
|
|
582
|
+
'Device not initialized. Call initDevice() first. ' +
|
|
583
|
+
'deviceState=' + JSON.stringify(diagnostics)
|
|
584
|
+
);
|
|
492
585
|
}
|
|
493
586
|
return { ...kernelCapabilities };
|
|
494
587
|
}
|
|
495
588
|
|
|
496
589
|
|
|
497
590
|
export function getDevice() {
|
|
591
|
+
hydrateDeviceState();
|
|
498
592
|
return gpuDevice;
|
|
499
593
|
}
|
|
500
594
|
|
|
501
595
|
export function getDeviceEpoch() {
|
|
596
|
+
hydrateDeviceState();
|
|
502
597
|
return deviceEpoch;
|
|
503
598
|
}
|
|
504
599
|
|
|
505
600
|
|
|
506
601
|
export function getPlatformConfig() {
|
|
602
|
+
hydrateDeviceState();
|
|
507
603
|
return resolvedPlatformConfig;
|
|
508
604
|
}
|
|
509
605
|
|
|
510
606
|
export function getLastDeviceLossInfo() {
|
|
607
|
+
hydrateDeviceState();
|
|
511
608
|
return lastDeviceLossInfo ? { ...lastDeviceLossInfo } : null;
|
|
512
609
|
}
|
|
513
610
|
|
|
@@ -519,6 +616,7 @@ export function resetDeviceState() {
|
|
|
519
616
|
|
|
520
617
|
|
|
521
618
|
export function destroyDevice() {
|
|
619
|
+
hydrateDeviceState();
|
|
522
620
|
if (gpuDevice) {
|
|
523
621
|
gpuDevice.destroy();
|
|
524
622
|
clearActiveDeviceState();
|
|
@@ -528,6 +626,7 @@ export function destroyDevice() {
|
|
|
528
626
|
|
|
529
627
|
|
|
530
628
|
export function hasFeature(feature) {
|
|
629
|
+
hydrateDeviceState();
|
|
531
630
|
if (!gpuDevice) {
|
|
532
631
|
return false;
|
|
533
632
|
}
|
|
@@ -536,6 +635,7 @@ export function hasFeature(feature) {
|
|
|
536
635
|
|
|
537
636
|
|
|
538
637
|
export function getDeviceLimits() {
|
|
638
|
+
hydrateDeviceState();
|
|
539
639
|
if (!gpuDevice) {
|
|
540
640
|
return null;
|
|
541
641
|
}
|
|
@@ -83,12 +83,9 @@ export function generateCacheKey(kernelName, inputSizes) {
|
|
|
83
83
|
|
|
84
84
|
|
|
85
85
|
export function loadCache(capabilities) {
|
|
86
|
-
// Clear stale cache when the device epoch has advanced (device reset/loss).
|
|
87
86
|
const currentEpoch = getDeviceEpoch();
|
|
88
87
|
if (cacheDeviceEpoch !== -1 && cacheDeviceEpoch !== currentEpoch) {
|
|
89
|
-
log.debug('KernelTuner', 'Device epoch changed (' + cacheDeviceEpoch + ' -> ' + currentEpoch + '),
|
|
90
|
-
cacheDeviceEpoch = currentEpoch;
|
|
91
|
-
return new Map();
|
|
88
|
+
log.debug('KernelTuner', 'Device epoch changed (' + cacheDeviceEpoch + ' -> ' + currentEpoch + '), reloading tuner cache');
|
|
92
89
|
}
|
|
93
90
|
cacheDeviceEpoch = currentEpoch;
|
|
94
91
|
|
|
@@ -146,4 +143,3 @@ export function clearCacheStorage(capabilities) {
|
|
|
146
143
|
localStorage.removeItem(getTunerConfig().cacheKeyPrefix + signature);
|
|
147
144
|
}
|
|
148
145
|
|
|
149
|
-
|