doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
--- src/gpu/kernels/attention_streaming.wgsl
|
|
2
2
|
+++ src/gpu/kernels/attention_streaming_f16.wgsl
|
|
3
|
+
@@ -6,6 +6,8 @@
|
|
4
|
+
// reads K/V directly from storage. Slower but compatible.
|
|
5
|
+
|
|
6
|
+
+enable f16;
|
|
7
|
+
+
|
|
8
|
+
override WORKGROUP_SIZE: u32 = 1u;
|
|
9
|
+
const MAX_HEAD_DIM: u32 = 512u;
|
|
3
10
|
@@ -30,10 +30,10 @@
|
|
4
11
|
}
|
|
5
12
|
|
|
@@ -14,7 +21,6 @@
|
|
|
14
21
|
+@group(0) @binding(4) var<storage, read_write> output: array<f16>;
|
|
15
22
|
@group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
|
|
16
23
|
@group(0) @binding(6) var<storage, read> page_table: array<u32>;
|
|
17
|
-
|
|
18
24
|
@@ -108,7 +108,7 @@
|
|
19
25
|
var q_local: array<f32, MAX_HEAD_DIM>;
|
|
20
26
|
let q_offset = query_pos * num_heads * head_dim + head_idx * head_dim;
|
|
@@ -24,3 +30,42 @@
|
|
|
24
30
|
}
|
|
25
31
|
|
|
26
32
|
var max_score: f32 = -3.402823e+38;
|
|
33
|
+
@@ -121,7 +121,7 @@
|
|
34
|
+
let k_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
|
|
35
|
+
var dot: f32 = 0.0;
|
|
36
|
+
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
37
|
+
- dot = dot + q_local[d] * K[k_offset + d];
|
|
38
|
+
+ dot = dot + q_local[d] * f32(K[k_offset + d]);
|
|
39
|
+
}
|
|
40
|
+
dot = dot * scale;
|
|
41
|
+
// Gemma 2 attention softcapping
|
|
42
|
+
@@ -144,7 +144,7 @@
|
|
43
|
+
let v_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
|
|
44
|
+
var dot: f32 = 0.0;
|
|
45
|
+
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
46
|
+
- dot = dot + q_local[d] * K[k_offset + d];
|
|
47
|
+
+ dot = dot + q_local[d] * f32(K[k_offset + d]);
|
|
48
|
+
}
|
|
49
|
+
dot = dot * scale;
|
|
50
|
+
// Gemma 2 attention softcapping
|
|
51
|
+
@@ -157,7 +157,7 @@
|
|
52
|
+
let w = exp(dot - max_score);
|
|
53
|
+
sum_exp = sum_exp + w;
|
|
54
|
+
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
55
|
+
- acc[d] = acc[d] + w * V[v_offset + d];
|
|
56
|
+
+ acc[d] = acc[d] + w * f32(V[v_offset + d]);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
@@ -165,12 +165,12 @@
|
|
60
|
+
if (sum_exp <= 0.0) {
|
|
61
|
+
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
62
|
+
- output[out_offset + d] = 0.0;
|
|
63
|
+
+ output[out_offset + d] = f16(0.0);
|
|
64
|
+
}
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
68
|
+
- output[out_offset + d] = acc[d] / sum_exp;
|
|
69
|
+
+ output[out_offset + d] = f16(acc[d] / sum_exp);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
@@ -39,23 +39,3 @@
|
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
42
|
-
@@ -46,19 +46,19 @@
|
|
43
|
-
let token_idx = idx / dim;
|
|
44
|
-
let dim_idx = idx % dim;
|
|
45
|
-
let row_base = token_idx * dim * 2u;
|
|
46
|
-
let g = input[row_base + dim_idx];
|
|
47
|
-
let up = input[row_base + dim + dim_idx];
|
|
48
|
-
output[idx] = gelu(g) * up;
|
|
49
|
-
return;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
if (HAS_GATE) {
|
|
53
|
-
let up = input[idx];
|
|
54
|
-
let g = gate[idx];
|
|
55
|
-
output[idx] = gelu(g) * up;
|
|
56
|
-
return;
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
let x = input[idx];
|
|
60
|
-
output[idx] = gelu(x);
|
|
61
|
-
}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
--- src/gpu/kernels/residual.wgsl
|
|
2
|
-
+++ src/gpu/kernels/residual_f16.wgsl
|
|
3
|
-
@@ -1,
|
|
1
|
+
--- src/gpu/kernels/residual.wgsl 2026-06-08 16:13:27.295360931 -0400
|
|
2
|
+
+++ src/gpu/kernels/residual_f16.wgsl 2026-06-08 16:32:38.676137468 -0400
|
|
3
|
+
@@ -1,12 +1,16 @@
|
|
4
4
|
-// residual.wgsl
|
|
5
|
+
+// AUTO-GENERATED from src/gpu/kernels/residual.wgsl.
|
|
6
|
+
+// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
5
7
|
+// residual_f16.wgsl
|
|
6
8
|
|
|
7
9
|
/**
|
|
@@ -9,14 +11,15 @@
|
|
|
9
11
|
+ * Residual Add Kernel (F16)
|
|
10
12
|
*
|
|
11
13
|
* Performs element-wise addition for residual connections.
|
|
12
|
-
* output = a + b
|
|
14
|
+
* output = (a + b) * scale
|
|
13
15
|
*/
|
|
14
|
-
+
|
|
15
|
-
+enable f16;
|
|
16
16
|
|
|
17
|
+
+enable f16;
|
|
18
|
+
+
|
|
17
19
|
struct Uniforms {
|
|
18
20
|
size: u32, // Total number of elements
|
|
19
|
-
|
|
21
|
+
scale: f32, // Output scale for main/add_vec4; residual scale for add_scaled
|
|
22
|
+
@@ -17,9 +21,9 @@
|
|
20
23
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
21
24
|
|
|
22
25
|
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
@@ -29,13 +32,12 @@
|
|
|
29
32
|
|
|
30
33
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
31
34
|
fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
|
32
|
-
@@ -28,29 +
|
|
35
|
+
@@ -28,29 +32,5 @@
|
|
33
36
|
if (idx >= u.size) {
|
|
34
37
|
return;
|
|
35
38
|
}
|
|
36
|
-
- output[idx] = a[idx] + b[idx];
|
|
37
|
-
|
|
38
|
-
}
|
|
39
|
+
- output[idx] = (a[idx] + b[idx]) * u.scale;
|
|
40
|
+
-}
|
|
39
41
|
-
|
|
40
42
|
-// In-place version: output = output + b
|
|
41
43
|
-// Note: Caller should copy 'a' to 'output' first, then call this kernel
|
|
@@ -59,4 +61,5 @@
|
|
|
59
61
|
- return;
|
|
60
62
|
- }
|
|
61
63
|
- output[idx] = a[idx] + u.scale * b[idx];
|
|
62
|
-
|
|
64
|
+
+ output[idx] = f16(f32(a[idx] + b[idx]) * u.scale);
|
|
65
|
+
}
|
|
@@ -1,7 +1,11 @@
|
|
|
1
|
-
--- src/gpu/kernels/rmsnorm.wgsl
|
|
2
|
-
+++ src/gpu/kernels/rmsnorm_f16.wgsl
|
|
3
|
-
@@ -1,46 +1,
|
|
1
|
+
--- src/gpu/kernels/rmsnorm.wgsl 2026-06-08 16:21:47.561276560 -0400
|
|
2
|
+
+++ src/gpu/kernels/rmsnorm_f16.wgsl 2026-06-08 16:32:38.675963250 -0400
|
|
3
|
+
@@ -1,46 +1,28 @@
|
|
4
4
|
-// RMSNorm Kernel with Fused Residual Add
|
|
5
|
+
+// AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
|
|
6
|
+
+// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
7
|
+
+// AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
|
|
8
|
+
+// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
5
9
|
//
|
|
6
10
|
-// RMSNorm(x) = x / sqrt(mean(x^2) + eps) * weight
|
|
7
11
|
-//
|
|
@@ -27,7 +31,7 @@
|
|
|
27
31
|
+// RMSNorm(x) = x / sqrt(mean(x^2) + eps) * weight
|
|
28
32
|
+
|
|
29
33
|
+enable f16;
|
|
30
|
-
|
|
34
|
+
|
|
31
35
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
32
36
|
const MAX_WORKGROUP_SIZE: u32 = 256u;
|
|
33
37
|
-
|
|
@@ -37,7 +41,7 @@
|
|
|
37
41
|
override PRE_RESIDUAL: bool = false; // Add residual BEFORE normalization: rmsnorm(input + residual)
|
|
38
42
|
override OUTPUT_PRENORM: bool = false; // Write pre-norm sum to residual_sum_output binding
|
|
39
43
|
override WEIGHT_IS_F16: bool = false; // Weight buffer packed as f16 pairs
|
|
40
|
-
|
|
44
|
+
|
|
41
45
|
-const MAX_SUBGROUPS: u32 = 32u; // Support up to 32 subgroups per workgroup
|
|
42
46
|
-const MAX_CACHE_SIZE: u32 = 4608u; // Supports hiddenSize up to 1152 with 4 elements/thread
|
|
43
47
|
-
|
|
@@ -53,11 +57,11 @@
|
|
|
53
57
|
+ eps: f32, // Epsilon for numerical stability
|
|
54
58
|
+ has_residual: u32, // 1 if residual input provided, 0 otherwise
|
|
55
59
|
token_stride: u32, // Workgroup rows per dispatch row
|
|
56
|
-
|
|
60
|
+
output_scale: f32, // Output epilogue scale
|
|
57
61
|
_pad1: u32,
|
|
58
|
-
@@ -48,47 +
|
|
62
|
+
@@ -48,47 +30,32 @@
|
|
59
63
|
}
|
|
60
|
-
|
|
64
|
+
|
|
61
65
|
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
62
66
|
-@group(0) @binding(1) var<storage, read> input: array<f32>;
|
|
63
67
|
-@group(0) @binding(2) var<storage, read> weight: array<u32>; // [size] as f32 or packed f16
|
|
@@ -79,7 +83,7 @@
|
|
|
79
83
|
+@group(0) @binding(3) var<storage, read_write> output: array<f16>;
|
|
80
84
|
+@group(0) @binding(4) var<storage, read> residual: array<f16>; // Optional residual
|
|
81
85
|
+@group(0) @binding(5) var<storage, read_write> residual_sum_output: array<f16>; // Pre-norm sum output (PRE_RESIDUAL + OUTPUT_PRENORM)
|
|
82
|
-
|
|
86
|
+
|
|
83
87
|
-// Apply weight with optional offset (Gemma uses 1 + weight pattern)
|
|
84
88
|
-fn apply_weight(w: f32) -> f32 {
|
|
85
89
|
+var<workgroup> shared_sum: array<f16, MAX_WORKGROUP_SIZE>;
|
|
@@ -93,7 +97,7 @@
|
|
|
93
97
|
}
|
|
94
98
|
+ return w;
|
|
95
99
|
}
|
|
96
|
-
|
|
100
|
+
|
|
97
101
|
-fn load_weight(idx: u32) -> f32 {
|
|
98
102
|
+fn load_weight(idx: u32) -> f16 {
|
|
99
103
|
if (WEIGHT_IS_F16) {
|
|
@@ -110,26 +114,32 @@
|
|
|
110
114
|
- return u.has_residual != 0u;
|
|
111
115
|
+ return f16(bitcast<f32>(weight[idx]));
|
|
112
116
|
}
|
|
113
|
-
|
|
117
|
+
|
|
114
118
|
// Load input value, fusing residual add when PRE_RESIDUAL is active
|
|
115
119
|
-fn load_input(base_offset: u32, idx: u32) -> f32 {
|
|
116
120
|
+fn load_input(base_offset: u32, idx: u32) -> f16 {
|
|
117
121
|
let x = input[base_offset + idx];
|
|
118
122
|
if (PRE_RESIDUAL) {
|
|
119
123
|
return x + residual[base_offset + idx];
|
|
120
|
-
@@ -97,
|
|
124
|
+
@@ -97,24 +64,16 @@
|
|
121
125
|
}
|
|
122
|
-
|
|
126
|
+
|
|
123
127
|
// Write pre-norm sum when OUTPUT_PRENORM is active (for downstream residual reuse)
|
|
124
128
|
-fn write_prenorm(base_offset: u32, idx: u32, val: f32) {
|
|
125
129
|
+fn write_prenorm(base_offset: u32, idx: u32, val: f16) {
|
|
126
130
|
if (OUTPUT_PRENORM && PRE_RESIDUAL) {
|
|
127
131
|
residual_sum_output[base_offset + idx] = val;
|
|
128
132
|
}
|
|
129
|
-
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
-fn apply_output_scale(value: f32) -> f32 {
|
|
136
|
+
- return value * u.output_scale;
|
|
137
|
+
-}
|
|
138
|
+
-
|
|
139
|
+
fn token_index(wg_id: vec3<u32>) -> u32 {
|
|
130
140
|
return wg_id.y * max(u.token_stride, 1u) + wg_id.x;
|
|
131
141
|
}
|
|
132
|
-
|
|
142
|
+
|
|
133
143
|
-// =============================================================================
|
|
134
144
|
-// Main Entry Point
|
|
135
145
|
-// =============================================================================
|
|
@@ -137,15 +147,15 @@
|
|
|
137
147
|
// Main RMSNorm kernel - one workgroup per token
|
|
138
148
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
139
149
|
fn main(
|
|
140
|
-
@@ -
|
|
141
|
-
|
|
150
|
+
@@ -132,151 +91,44 @@
|
|
151
|
+
|
|
142
152
|
let base_offset = token_idx * size;
|
|
143
|
-
|
|
153
|
+
|
|
144
154
|
- // Each thread computes partial sum of squares
|
|
145
155
|
- // When PRE_RESIDUAL is active, sum is computed on (input + residual)
|
|
146
156
|
- var local_sum_sq: f32 = 0.0;
|
|
147
157
|
let elements_per_thread = (size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE;
|
|
148
|
-
|
|
158
|
+
|
|
149
159
|
+ var local_max: f16 = f16(0.0);
|
|
150
160
|
for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
|
|
151
161
|
let idx = thread_idx * elements_per_thread + i;
|
|
@@ -155,12 +165,12 @@
|
|
|
155
165
|
+ local_max = max(local_max, abs(load_input(base_offset, idx)));
|
|
156
166
|
}
|
|
157
167
|
}
|
|
158
|
-
|
|
168
|
+
|
|
159
169
|
- // Store local sum for reduction
|
|
160
170
|
- shared_sum[thread_idx] = local_sum_sq;
|
|
161
171
|
+ shared_sum[thread_idx] = local_max;
|
|
162
172
|
workgroupBarrier();
|
|
163
|
-
|
|
173
|
+
|
|
164
174
|
- // Parallel reduction to compute total sum of squares
|
|
165
175
|
for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
|
|
166
176
|
if (thread_idx < stride) {
|
|
@@ -169,7 +179,7 @@
|
|
|
169
179
|
}
|
|
170
180
|
workgroupBarrier();
|
|
171
181
|
}
|
|
172
|
-
|
|
182
|
+
|
|
173
183
|
- // Compute RMS
|
|
174
184
|
- let mean_sq = shared_sum[0] / f32(size);
|
|
175
185
|
- let rms = sqrt(mean_sq + u.eps);
|
|
@@ -177,7 +187,7 @@
|
|
|
177
187
|
-
|
|
178
188
|
+ let row_scale = select(f16(1.0), shared_sum[0], shared_sum[0] > f16(0.0));
|
|
179
189
|
workgroupBarrier();
|
|
180
|
-
|
|
190
|
+
|
|
181
191
|
- // Apply normalization and weight
|
|
182
192
|
- for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
|
|
183
193
|
- let idx = thread_idx * elements_per_thread + i;
|
|
@@ -195,7 +205,7 @@
|
|
|
195
205
|
- // Write pre-norm sum for downstream residual reuse
|
|
196
206
|
- write_prenorm(base_offset, idx, x);
|
|
197
207
|
-
|
|
198
|
-
- output[base_offset + idx] = result;
|
|
208
|
+
- output[base_offset + idx] = apply_output_scale(result);
|
|
199
209
|
- }
|
|
200
210
|
- }
|
|
201
211
|
-}
|
|
@@ -249,7 +259,7 @@
|
|
|
249
259
|
- result = result + residual[base_offset + thread_idx];
|
|
250
260
|
- }
|
|
251
261
|
- write_prenorm(base_offset, thread_idx, x);
|
|
252
|
-
- output[base_offset + thread_idx] = result;
|
|
262
|
+
- output[base_offset + thread_idx] = apply_output_scale(result);
|
|
253
263
|
- }
|
|
254
264
|
-}
|
|
255
265
|
-
|
|
@@ -291,27 +301,27 @@
|
|
|
291
301
|
+ local_sum_sq = local_sum_sq + (x * x) * inv_size;
|
|
292
302
|
}
|
|
293
303
|
}
|
|
294
|
-
|
|
304
|
+
|
|
295
305
|
+ // Store local sum for reduction
|
|
296
306
|
shared_sum[thread_idx] = local_sum_sq;
|
|
297
307
|
workgroupBarrier();
|
|
298
|
-
|
|
308
|
+
|
|
299
309
|
+ // Parallel reduction to compute total sum of squares
|
|
300
310
|
for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
|
|
301
311
|
if (thread_idx < stride) {
|
|
302
312
|
shared_sum[thread_idx] = shared_sum[thread_idx] + shared_sum[thread_idx + stride];
|
|
303
|
-
@@ -
|
|
313
|
+
@@ -284,117 +136,38 @@
|
|
304
314
|
workgroupBarrier();
|
|
305
315
|
}
|
|
306
|
-
|
|
316
|
+
|
|
307
317
|
- let mean_sq = shared_sum[0] / f32(size);
|
|
308
318
|
- let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
|
|
309
319
|
+ let mean_sq = shared_sum[0];
|
|
310
320
|
+ let eps_scaled = f16(u.eps) / (row_scale * row_scale);
|
|
311
321
|
+ let inv_rms = (f16(1.0) / row_scale) / sqrt(mean_sq + eps_scaled);
|
|
312
|
-
|
|
322
|
+
|
|
313
323
|
workgroupBarrier();
|
|
314
|
-
|
|
324
|
+
|
|
315
325
|
- // Second pass: normalize cached input
|
|
316
326
|
- for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
|
|
317
327
|
- let idx = thread_idx * elements_per_thread + i;
|
|
@@ -322,7 +332,7 @@
|
|
|
322
332
|
- result = result + residual[base_offset + idx];
|
|
323
333
|
- }
|
|
324
334
|
- write_prenorm(base_offset, idx, x);
|
|
325
|
-
- output[base_offset + idx] = result;
|
|
335
|
+
- output[base_offset + idx] = apply_output_scale(result);
|
|
326
336
|
- }
|
|
327
337
|
- }
|
|
328
338
|
-}
|
|
@@ -400,21 +410,22 @@
|
|
|
400
410
|
let x = load_input(base_offset, idx);
|
|
401
411
|
+
|
|
402
412
|
var result = x * inv_rms * apply_weight(load_weight(idx));
|
|
403
|
-
|
|
413
|
+
|
|
404
414
|
- if (should_add_residual()) {
|
|
405
415
|
+ // Add residual AFTER normalization
|
|
406
416
|
+ if (u.has_residual == 1u) {
|
|
407
417
|
result = result + residual[base_offset + idx];
|
|
408
418
|
}
|
|
409
|
-
|
|
419
|
+
|
|
410
420
|
+ // Write pre-norm sum for downstream residual reuse
|
|
411
421
|
write_prenorm(base_offset, idx, x);
|
|
422
|
+
- output[base_offset + idx] = apply_output_scale(result);
|
|
412
423
|
+
|
|
413
|
-
|
|
424
|
+
+ output[base_offset + idx] = f16(f32(result) * u.output_scale);
|
|
414
425
|
}
|
|
415
426
|
}
|
|
416
427
|
}
|
|
417
|
-
|
|
428
|
+
|
|
418
429
|
-// Subgroup-accelerated RMSNorm for small hidden sizes (size <= 256)
|
|
419
430
|
-// Each thread handles one element, uses subgroup reduction
|
|
420
431
|
+// Optimized version for hidden size <= WORKGROUP_SIZE (single pass)
|
|
@@ -429,13 +440,13 @@
|
|
|
429
440
|
) {
|
|
430
441
|
let token_idx = token_index(wg_id);
|
|
431
442
|
let thread_idx = local_id.x;
|
|
432
|
-
@@ -
|
|
443
|
+
@@ -405,48 +178,49 @@
|
|
433
444
|
}
|
|
434
|
-
|
|
445
|
+
|
|
435
446
|
let base_offset = token_idx * size;
|
|
436
447
|
- let subgroup_id = thread_idx / sg_size;
|
|
437
448
|
- let num_subgroups = (WORKGROUP_SIZE + sg_size - 1u) / sg_size;
|
|
438
|
-
|
|
449
|
+
|
|
439
450
|
- // Each thread handles one element
|
|
440
451
|
- var x: f32 = 0.0;
|
|
441
452
|
- var x_sq: f32 = 0.0;
|
|
@@ -445,12 +456,12 @@
|
|
|
445
456
|
x = load_input(base_offset, thread_idx);
|
|
446
457
|
- x_sq = x * x;
|
|
447
458
|
}
|
|
448
|
-
|
|
459
|
+
|
|
449
460
|
- // Phase 1: Subgroup reduction
|
|
450
461
|
- let sg_sum = subgroupAdd(x_sq);
|
|
451
462
|
+ shared_sum[thread_idx] = abs(x);
|
|
452
463
|
+ workgroupBarrier();
|
|
453
|
-
|
|
464
|
+
|
|
454
465
|
- // Phase 2: Store to shared memory
|
|
455
466
|
- if (sg_lane == 0u && subgroup_id < num_subgroups) {
|
|
456
467
|
- sg_partial_sums[subgroup_id] = sg_sum;
|
|
@@ -466,7 +477,7 @@
|
|
|
466
477
|
+ let inv_size = f16(1.0) / f16(size);
|
|
467
478
|
+ shared_sum[thread_idx] = (x_scaled * x_scaled) * inv_size;
|
|
468
479
|
workgroupBarrier();
|
|
469
|
-
|
|
480
|
+
|
|
470
481
|
- // Phase 3: Final reduction - thread 0 sums, then broadcast via shared memory
|
|
471
482
|
- if (thread_idx == 0u) {
|
|
472
483
|
- var sum: f32 = 0.0;
|
|
@@ -482,14 +493,14 @@
|
|
|
482
493
|
}
|
|
483
494
|
- workgroupBarrier();
|
|
484
495
|
- let total_sum = sg_partial_sums[0];
|
|
485
|
-
|
|
496
|
+
|
|
486
497
|
// Compute inverse RMS
|
|
487
498
|
- let mean_sq = total_sum / f32(size);
|
|
488
499
|
- let inv_rms = 1.0 / sqrt(mean_sq + u.eps);
|
|
489
500
|
+ let mean_sq = shared_sum[0];
|
|
490
501
|
+ let eps_scaled = f16(u.eps) / (row_scale * row_scale);
|
|
491
502
|
+ let inv_rms = (f16(1.0) / row_scale) / sqrt(mean_sq + eps_scaled);
|
|
492
|
-
|
|
503
|
+
|
|
493
504
|
// Apply normalization
|
|
494
505
|
if (thread_idx < size) {
|
|
495
506
|
var result = x * inv_rms * apply_weight(load_weight(thread_idx));
|
|
@@ -498,3 +509,7 @@
|
|
|
498
509
|
result = result + residual[base_offset + thread_idx];
|
|
499
510
|
}
|
|
500
511
|
write_prenorm(base_offset, thread_idx, x);
|
|
512
|
+
- output[base_offset + thread_idx] = apply_output_scale(result);
|
|
513
|
+
+ output[base_offset + thread_idx] = f16(f32(result) * u.output_scale);
|
|
514
|
+
}
|
|
515
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export interface DiffusionGemmaCanvasStatsOptions {
|
|
2
|
+
canvasLength: number;
|
|
3
|
+
vocabSize: number;
|
|
4
|
+
temperature: number;
|
|
5
|
+
padTokenId: number | null;
|
|
6
|
+
logitSoftcap: number;
|
|
7
|
+
argmaxBuffer?: GPUBuffer | null;
|
|
8
|
+
entropyBuffer?: GPUBuffer | null;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface DiffusionGemmaCanvasStatsResult {
|
|
12
|
+
argmaxBuffer: GPUBuffer;
|
|
13
|
+
entropyBuffer: GPUBuffer;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function runDiffusionGemmaCanvasStats(
|
|
17
|
+
logitsBuffer: GPUBuffer,
|
|
18
|
+
options: DiffusionGemmaCanvasStatsOptions
|
|
19
|
+
): Promise<DiffusionGemmaCanvasStatsResult>;
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { getDevice } from '../device.js';
|
|
2
|
+
import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
|
|
3
|
+
import {
|
|
4
|
+
createPipeline,
|
|
5
|
+
createUniformBufferWithView,
|
|
6
|
+
createBindGroupWithValidation,
|
|
7
|
+
} from './utils.js';
|
|
8
|
+
import { dispatchKernel } from './dispatch.js';
|
|
9
|
+
|
|
10
|
+
const UNIFORM_SIZE = 32;
|
|
11
|
+
|
|
12
|
+
function assertPositiveInteger(value, label) {
|
|
13
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
14
|
+
throw new Error(`[DiffusionGemmaSampling] ${label} must be a positive integer.`);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function assertFinitePositiveNumber(value, label) {
|
|
19
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) {
|
|
20
|
+
throw new Error(`[DiffusionGemmaSampling] ${label} must be a positive finite number.`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function assertFiniteNonNegativeNumber(value, label) {
|
|
25
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
|
26
|
+
throw new Error(`[DiffusionGemmaSampling] ${label} must be a non-negative finite number.`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function createStatsUniformBuffer(device, options) {
|
|
31
|
+
return createUniformBufferWithView(
|
|
32
|
+
'diffusion_gemma_sampling_uniforms',
|
|
33
|
+
UNIFORM_SIZE,
|
|
34
|
+
(view) => {
|
|
35
|
+
view.setUint32(0, options.vocabSize, true);
|
|
36
|
+
view.setUint32(4, options.canvasLength, true);
|
|
37
|
+
view.setUint32(8, options.padTokenId == null ? 0xffffffff : options.padTokenId, true);
|
|
38
|
+
view.setUint32(12, 0, true);
|
|
39
|
+
view.setFloat32(16, options.temperature, true);
|
|
40
|
+
view.setFloat32(20, options.logitSoftcap, true);
|
|
41
|
+
view.setFloat32(24, 0, true);
|
|
42
|
+
view.setFloat32(28, 0, true);
|
|
43
|
+
},
|
|
44
|
+
null,
|
|
45
|
+
device
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export async function runDiffusionGemmaCanvasStats(logitsBuffer, options = {}) {
|
|
50
|
+
const device = getDevice();
|
|
51
|
+
if (!device) {
|
|
52
|
+
throw new Error('[DiffusionGemmaSampling] GPU device is not initialized.');
|
|
53
|
+
}
|
|
54
|
+
if (!logitsBuffer) {
|
|
55
|
+
throw new Error('[DiffusionGemmaSampling] logitsBuffer is required.');
|
|
56
|
+
}
|
|
57
|
+
assertPositiveInteger(options.canvasLength, 'canvasLength');
|
|
58
|
+
assertPositiveInteger(options.vocabSize, 'vocabSize');
|
|
59
|
+
assertFinitePositiveNumber(options.temperature, 'temperature');
|
|
60
|
+
assertFiniteNonNegativeNumber(options.logitSoftcap, 'logitSoftcap');
|
|
61
|
+
if (options.padTokenId != null && (!Number.isInteger(options.padTokenId) || options.padTokenId < 0)) {
|
|
62
|
+
throw new Error('[DiffusionGemmaSampling] padTokenId must be null or a non-negative integer.');
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const tokenBytes = options.canvasLength * Uint32Array.BYTES_PER_ELEMENT;
|
|
66
|
+
const entropyBytes = options.canvasLength * Float32Array.BYTES_PER_ELEMENT;
|
|
67
|
+
const argmaxBuffer = options.argmaxBuffer ?? acquireBuffer(tokenBytes, undefined, 'diffusion_gemma_argmax_canvas');
|
|
68
|
+
const entropyBuffer = options.entropyBuffer ?? acquireBuffer(entropyBytes, undefined, 'diffusion_gemma_entropy_canvas');
|
|
69
|
+
const ownsArgmax = options.argmaxBuffer == null;
|
|
70
|
+
const ownsEntropy = options.entropyBuffer == null;
|
|
71
|
+
let uniformBuffer = null;
|
|
72
|
+
let completed = false;
|
|
73
|
+
|
|
74
|
+
try {
|
|
75
|
+
if (argmaxBuffer.size < tokenBytes) {
|
|
76
|
+
throw new Error('[DiffusionGemmaSampling] argmaxBuffer is too small.');
|
|
77
|
+
}
|
|
78
|
+
if (entropyBuffer.size < entropyBytes) {
|
|
79
|
+
throw new Error('[DiffusionGemmaSampling] entropyBuffer is too small.');
|
|
80
|
+
}
|
|
81
|
+
const pipeline = await createPipeline('diffusion_gemma_sampling', 'entropy_stats');
|
|
82
|
+
uniformBuffer = createStatsUniformBuffer(device, options);
|
|
83
|
+
const bindGroup = await createBindGroupWithValidation(device, {
|
|
84
|
+
label: 'diffusion_gemma_sampling_bind_group',
|
|
85
|
+
layout: pipeline.getBindGroupLayout(0),
|
|
86
|
+
entries: [
|
|
87
|
+
{ binding: 0, resource: { buffer: uniformBuffer } },
|
|
88
|
+
{ binding: 1, resource: { buffer: logitsBuffer } },
|
|
89
|
+
{ binding: 2, resource: { buffer: argmaxBuffer } },
|
|
90
|
+
{ binding: 3, resource: { buffer: entropyBuffer } },
|
|
91
|
+
],
|
|
92
|
+
}, 'diffusion_gemma_sampling/entropy_stats');
|
|
93
|
+
|
|
94
|
+
dispatchKernel(null, pipeline, bindGroup, options.canvasLength, 'diffusion_gemma_entropy_stats');
|
|
95
|
+
completed = true;
|
|
96
|
+
return { argmaxBuffer, entropyBuffer };
|
|
97
|
+
} finally {
|
|
98
|
+
uniformBuffer?.destroy?.();
|
|
99
|
+
if (!completed) {
|
|
100
|
+
if (ownsArgmax) releaseBuffer(argmaxBuffer);
|
|
101
|
+
if (ownsEntropy) releaseBuffer(entropyBuffer);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|