doppler-gpu 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +67 -34
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.js +5 -1
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/version.js +1 -1
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
const UINT32_SCALE = 1 / 0x100000000;
|
|
2
|
+
|
|
3
|
+
export function createSeededRandom(seed) {
|
|
4
|
+
if (!Number.isInteger(seed)) {
|
|
5
|
+
throw new Error('DiffusionGemma seeded random requires an integer seed.');
|
|
6
|
+
}
|
|
7
|
+
let state = seed >>> 0;
|
|
8
|
+
return () => {
|
|
9
|
+
state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
|
|
10
|
+
return state * UINT32_SCALE;
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function assertCanvas(canvas, canvasLength, label) {
|
|
15
|
+
if (!Array.isArray(canvas) && !ArrayBuffer.isView(canvas)) {
|
|
16
|
+
throw new Error(`${label} must be an array or typed array.`);
|
|
17
|
+
}
|
|
18
|
+
if (canvas.length !== canvasLength) {
|
|
19
|
+
throw new Error(`${label} length ${canvas.length} does not match canvasLength ${canvasLength}.`);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function randomToken(random, vocabSize) {
|
|
24
|
+
return Math.min(vocabSize - 1, Math.floor(random() * vocabSize));
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function normalizeTemperature(value) {
|
|
28
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) {
|
|
29
|
+
throw new Error(`DiffusionGemma temperature must be positive; got ${String(value)}.`);
|
|
30
|
+
}
|
|
31
|
+
return value;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function resolveDenoisingTemperature(config, step) {
|
|
35
|
+
const maxSteps = config.maxDenoisingSteps;
|
|
36
|
+
if (!Number.isInteger(step) || step < 1 || step > maxSteps) {
|
|
37
|
+
throw new Error(`DiffusionGemma denoising step must be in [1, ${maxSteps}], got ${String(step)}.`);
|
|
38
|
+
}
|
|
39
|
+
return normalizeTemperature(config.tMin + ((config.tMax - config.tMin) * (step / maxSteps)));
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function initializeCanvas(config, random) {
|
|
43
|
+
if (typeof random !== 'function') {
|
|
44
|
+
throw new Error('DiffusionGemma canvas initialization requires a random function.');
|
|
45
|
+
}
|
|
46
|
+
const canvas = new Int32Array(config.canvasLength);
|
|
47
|
+
for (let i = 0; i < canvas.length; i += 1) {
|
|
48
|
+
canvas[i] = randomToken(random, config.vocabSize);
|
|
49
|
+
}
|
|
50
|
+
return canvas;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function getLogitsView(logits, tokenIndex, vocabSize) {
|
|
54
|
+
const start = tokenIndex * vocabSize;
|
|
55
|
+
const end = start + vocabSize;
|
|
56
|
+
if (logits.length < end) {
|
|
57
|
+
throw new Error(
|
|
58
|
+
`DiffusionGemma logits length ${logits.length} is too small for token ${tokenIndex} and vocabSize ${vocabSize}.`
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
return logits.subarray(start, end);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function softmaxStats(tokenLogits, processedLogits, temperature, random) {
|
|
65
|
+
let max = -Infinity;
|
|
66
|
+
for (let i = 0; i < tokenLogits.length; i += 1) {
|
|
67
|
+
const value = tokenLogits[i] / temperature;
|
|
68
|
+
processedLogits[i] = value;
|
|
69
|
+
if (value > max) max = value;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
let sum = 0;
|
|
73
|
+
let weightedLogitSum = 0;
|
|
74
|
+
let argmaxToken = 0;
|
|
75
|
+
let argmaxValue = -Infinity;
|
|
76
|
+
const probs = new Float32Array(tokenLogits.length);
|
|
77
|
+
for (let i = 0; i < tokenLogits.length; i += 1) {
|
|
78
|
+
const scaled = processedLogits[i];
|
|
79
|
+
if (scaled > argmaxValue) {
|
|
80
|
+
argmaxValue = scaled;
|
|
81
|
+
argmaxToken = i;
|
|
82
|
+
}
|
|
83
|
+
const expValue = Math.exp(scaled - max);
|
|
84
|
+
probs[i] = expValue;
|
|
85
|
+
sum += expValue;
|
|
86
|
+
weightedLogitSum += expValue * scaled;
|
|
87
|
+
}
|
|
88
|
+
if (!Number.isFinite(sum) || sum <= 0) {
|
|
89
|
+
throw new Error('DiffusionGemma logits produced an invalid softmax denominator.');
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const draw = random() * sum;
|
|
93
|
+
let cumulative = 0;
|
|
94
|
+
let sampledToken = tokenLogits.length - 1;
|
|
95
|
+
for (let i = 0; i < probs.length; i += 1) {
|
|
96
|
+
cumulative += probs[i];
|
|
97
|
+
if (draw <= cumulative) {
|
|
98
|
+
sampledToken = i;
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const entropy = Math.log(sum) + max - (weightedLogitSum / sum);
|
|
104
|
+
return {
|
|
105
|
+
argmaxToken,
|
|
106
|
+
sampledToken,
|
|
107
|
+
entropy,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function applyEntropyBoundStep(currentCanvas, logits, config, options = {}) {
|
|
112
|
+
const random = options.random;
|
|
113
|
+
if (typeof random !== 'function') {
|
|
114
|
+
throw new Error('DiffusionGemma entropy-bound step requires a random function.');
|
|
115
|
+
}
|
|
116
|
+
assertCanvas(currentCanvas, config.canvasLength, 'currentCanvas');
|
|
117
|
+
if (!ArrayBuffer.isView(logits)) {
|
|
118
|
+
throw new Error('DiffusionGemma entropy-bound step requires typed-array logits.');
|
|
119
|
+
}
|
|
120
|
+
const temperature = normalizeTemperature(options.temperature);
|
|
121
|
+
const nextCanvas = new Int32Array(config.canvasLength);
|
|
122
|
+
const argmaxCanvas = new Int32Array(config.canvasLength);
|
|
123
|
+
const sampledCanvas = new Int32Array(config.canvasLength);
|
|
124
|
+
const entropies = new Float32Array(config.canvasLength);
|
|
125
|
+
const processedLogits = new Float32Array(config.canvasLength * config.vocabSize);
|
|
126
|
+
const entropyOrder = [];
|
|
127
|
+
|
|
128
|
+
for (let i = 0; i < config.canvasLength; i += 1) {
|
|
129
|
+
const stats = softmaxStats(
|
|
130
|
+
getLogitsView(logits, i, config.vocabSize),
|
|
131
|
+
getLogitsView(processedLogits, i, config.vocabSize),
|
|
132
|
+
temperature,
|
|
133
|
+
random
|
|
134
|
+
);
|
|
135
|
+
argmaxCanvas[i] = stats.argmaxToken;
|
|
136
|
+
sampledCanvas[i] = stats.sampledToken;
|
|
137
|
+
entropies[i] = stats.entropy;
|
|
138
|
+
entropyOrder.push({ index: i, entropy: stats.entropy });
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
entropyOrder.sort((a, b) => a.entropy - b.entropy || a.index - b.index);
|
|
142
|
+
const accepted = new Uint8Array(config.canvasLength);
|
|
143
|
+
let cumulativeEntropy = 0;
|
|
144
|
+
let acceptedCount = 0;
|
|
145
|
+
for (const entry of entropyOrder) {
|
|
146
|
+
if (cumulativeEntropy <= config.entropyBound) {
|
|
147
|
+
accepted[entry.index] = 1;
|
|
148
|
+
acceptedCount += 1;
|
|
149
|
+
}
|
|
150
|
+
cumulativeEntropy += entry.entropy;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
for (let i = 0; i < config.canvasLength; i += 1) {
|
|
154
|
+
nextCanvas[i] = accepted[i] ? sampledCanvas[i] : randomToken(random, config.vocabSize);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return {
|
|
158
|
+
canvas: nextCanvas,
|
|
159
|
+
argmaxCanvas,
|
|
160
|
+
sampledCanvas,
|
|
161
|
+
entropies,
|
|
162
|
+
processedLogits,
|
|
163
|
+
accepted,
|
|
164
|
+
acceptedCount,
|
|
165
|
+
meanEntropy: entropies.reduce((sum, value) => sum + value, 0) / entropies.length,
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export function applyEntropyBoundStatsStep(currentCanvas, stats, config, options = {}) {
|
|
170
|
+
const random = options.random;
|
|
171
|
+
if (typeof random !== 'function') {
|
|
172
|
+
throw new Error('DiffusionGemma entropy-bound stats step requires a random function.');
|
|
173
|
+
}
|
|
174
|
+
assertCanvas(currentCanvas, config.canvasLength, 'currentCanvas');
|
|
175
|
+
const argmaxCanvas = stats?.argmaxCanvas;
|
|
176
|
+
const entropies = stats?.entropies;
|
|
177
|
+
assertCanvas(argmaxCanvas, config.canvasLength, 'argmaxCanvas');
|
|
178
|
+
if (!(entropies instanceof Float32Array) || entropies.length !== config.canvasLength) {
|
|
179
|
+
throw new Error('DiffusionGemma entropy stats step requires entropies as Float32Array[canvasLength].');
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const nextCanvas = new Int32Array(config.canvasLength);
|
|
183
|
+
const accepted = new Uint8Array(config.canvasLength);
|
|
184
|
+
const entropyOrder = [];
|
|
185
|
+
for (let i = 0; i < config.canvasLength; i += 1) {
|
|
186
|
+
const entropy = entropies[i];
|
|
187
|
+
if (!Number.isFinite(entropy) || entropy < 0) {
|
|
188
|
+
throw new Error(`DiffusionGemma entropy stats step received invalid entropy at ${i}.`);
|
|
189
|
+
}
|
|
190
|
+
entropyOrder.push({ index: i, entropy });
|
|
191
|
+
}
|
|
192
|
+
entropyOrder.sort((a, b) => a.entropy - b.entropy || a.index - b.index);
|
|
193
|
+
|
|
194
|
+
let cumulativeEntropy = 0;
|
|
195
|
+
let acceptedCount = 0;
|
|
196
|
+
for (const entry of entropyOrder) {
|
|
197
|
+
if (cumulativeEntropy <= config.entropyBound) {
|
|
198
|
+
accepted[entry.index] = 1;
|
|
199
|
+
acceptedCount += 1;
|
|
200
|
+
}
|
|
201
|
+
cumulativeEntropy += entry.entropy;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
for (let i = 0; i < config.canvasLength; i += 1) {
|
|
205
|
+
nextCanvas[i] = accepted[i] ? argmaxCanvas[i] : randomToken(random, config.vocabSize);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
return {
|
|
209
|
+
canvas: nextCanvas,
|
|
210
|
+
argmaxCanvas: Int32Array.from(argmaxCanvas),
|
|
211
|
+
sampledCanvas: Int32Array.from(argmaxCanvas),
|
|
212
|
+
entropies,
|
|
213
|
+
processedLogits: null,
|
|
214
|
+
accepted,
|
|
215
|
+
acceptedCount,
|
|
216
|
+
meanEntropy: entropies.reduce((sum, value) => sum + value, 0) / entropies.length,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
export function updateStabilityState(previousArgmaxCanvas, argmaxCanvas, previousCounts, config) {
|
|
221
|
+
assertCanvas(argmaxCanvas, config.canvasLength, 'argmaxCanvas');
|
|
222
|
+
const counts = new Uint16Array(config.canvasLength);
|
|
223
|
+
if (config.stabilityThreshold === 0) {
|
|
224
|
+
counts.fill(0xffff);
|
|
225
|
+
return {
|
|
226
|
+
counts,
|
|
227
|
+
stableTokens: config.canvasLength,
|
|
228
|
+
allStable: true,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
let stableTokens = 0;
|
|
232
|
+
for (let i = 0; i < config.canvasLength; i += 1) {
|
|
233
|
+
const previousMatches = previousArgmaxCanvas && previousArgmaxCanvas[i] === argmaxCanvas[i];
|
|
234
|
+
counts[i] = previousMatches ? ((previousCounts?.[i] ?? 0) + 1) : 0;
|
|
235
|
+
if (counts[i] >= config.stabilityThreshold) {
|
|
236
|
+
stableTokens += 1;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
return {
|
|
240
|
+
counts,
|
|
241
|
+
stableTokens,
|
|
242
|
+
allStable: stableTokens === config.canvasLength,
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function releaseSelfConditioningState(state) {
|
|
247
|
+
if (state && !ArrayBuffer.isView(state) && typeof state.release === 'function') {
|
|
248
|
+
state.release();
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
export async function denoiseCanvas(config, options) {
|
|
253
|
+
const logitsProvider = options?.logitsProvider;
|
|
254
|
+
const random = options?.random;
|
|
255
|
+
if (typeof logitsProvider !== 'function') {
|
|
256
|
+
throw new Error('DiffusionGemma denoiseCanvas requires a logitsProvider function.');
|
|
257
|
+
}
|
|
258
|
+
if (typeof random !== 'function') {
|
|
259
|
+
throw new Error('DiffusionGemma denoiseCanvas requires a random function.');
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
let canvas = options.initialCanvas == null
|
|
263
|
+
? initializeCanvas(config, random)
|
|
264
|
+
: Int32Array.from(options.initialCanvas);
|
|
265
|
+
assertCanvas(canvas, config.canvasLength, 'initialCanvas');
|
|
266
|
+
let previousArgmaxCanvas = null;
|
|
267
|
+
let stabilityCounts = null;
|
|
268
|
+
let selfConditioningLogits = options.selfConditioningLogits ?? null;
|
|
269
|
+
let lastStep = null;
|
|
270
|
+
let stepsRun = 0;
|
|
271
|
+
|
|
272
|
+
for (let step = config.maxDenoisingSteps; step >= 1; step -= 1) {
|
|
273
|
+
stepsRun += 1;
|
|
274
|
+
const temperature = resolveDenoisingTemperature(config, step);
|
|
275
|
+
const logits = await logitsProvider({
|
|
276
|
+
canvas,
|
|
277
|
+
step,
|
|
278
|
+
temperature,
|
|
279
|
+
selfConditioningLogits,
|
|
280
|
+
canvasIndex: options.canvasIndex ?? 0,
|
|
281
|
+
inputIds: options.inputIds ?? null,
|
|
282
|
+
});
|
|
283
|
+
const stepResult = applyEntropyBoundStep(canvas, logits, config, {
|
|
284
|
+
temperature,
|
|
285
|
+
random,
|
|
286
|
+
});
|
|
287
|
+
const stability = updateStabilityState(
|
|
288
|
+
previousArgmaxCanvas,
|
|
289
|
+
stepResult.argmaxCanvas,
|
|
290
|
+
stabilityCounts,
|
|
291
|
+
config
|
|
292
|
+
);
|
|
293
|
+
|
|
294
|
+
canvas = stepResult.canvas;
|
|
295
|
+
previousArgmaxCanvas = stepResult.argmaxCanvas;
|
|
296
|
+
stabilityCounts = stability.counts;
|
|
297
|
+
selfConditioningLogits = stepResult.processedLogits;
|
|
298
|
+
lastStep = {
|
|
299
|
+
...stepResult,
|
|
300
|
+
step,
|
|
301
|
+
temperature,
|
|
302
|
+
stability,
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
if (stability.allStable && stepResult.meanEntropy < config.confidenceThreshold) {
|
|
306
|
+
break;
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
return {
|
|
311
|
+
canvas,
|
|
312
|
+
argmaxCanvas: previousArgmaxCanvas,
|
|
313
|
+
selfConditioningLogits,
|
|
314
|
+
lastStep,
|
|
315
|
+
stepsRun,
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
export async function denoiseCanvasWithStatsProvider(config, options) {
|
|
320
|
+
const statsProvider = options?.statsProvider;
|
|
321
|
+
const random = options?.random;
|
|
322
|
+
if (typeof statsProvider !== 'function') {
|
|
323
|
+
throw new Error('DiffusionGemma denoiseCanvasWithStatsProvider requires a statsProvider function.');
|
|
324
|
+
}
|
|
325
|
+
if (typeof random !== 'function') {
|
|
326
|
+
throw new Error('DiffusionGemma denoiseCanvasWithStatsProvider requires a random function.');
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
let canvas = options.initialCanvas == null
|
|
330
|
+
? initializeCanvas(config, random)
|
|
331
|
+
: Int32Array.from(options.initialCanvas);
|
|
332
|
+
assertCanvas(canvas, config.canvasLength, 'initialCanvas');
|
|
333
|
+
let previousArgmaxCanvas = null;
|
|
334
|
+
let stabilityCounts = null;
|
|
335
|
+
const initialSelfConditioningLogits = options.selfConditioningLogits ?? null;
|
|
336
|
+
let selfConditioningLogits = initialSelfConditioningLogits;
|
|
337
|
+
let lastStep = null;
|
|
338
|
+
let stepsRun = 0;
|
|
339
|
+
let completed = false;
|
|
340
|
+
|
|
341
|
+
try {
|
|
342
|
+
for (let step = config.maxDenoisingSteps; step >= 1; step -= 1) {
|
|
343
|
+
stepsRun += 1;
|
|
344
|
+
const temperature = resolveDenoisingTemperature(config, step);
|
|
345
|
+
const inputSelfConditioningLogits = selfConditioningLogits;
|
|
346
|
+
selfConditioningLogits = null;
|
|
347
|
+
const stats = await statsProvider({
|
|
348
|
+
canvas,
|
|
349
|
+
step,
|
|
350
|
+
temperature,
|
|
351
|
+
selfConditioningLogits: inputSelfConditioningLogits,
|
|
352
|
+
canvasIndex: options.canvasIndex ?? 0,
|
|
353
|
+
inputIds: options.inputIds ?? null,
|
|
354
|
+
});
|
|
355
|
+
selfConditioningLogits = stats.selfConditioningLogits ?? null;
|
|
356
|
+
const stepResult = applyEntropyBoundStatsStep(canvas, stats, config, {
|
|
357
|
+
temperature,
|
|
358
|
+
random,
|
|
359
|
+
});
|
|
360
|
+
const stability = updateStabilityState(
|
|
361
|
+
previousArgmaxCanvas,
|
|
362
|
+
stepResult.argmaxCanvas,
|
|
363
|
+
stabilityCounts,
|
|
364
|
+
config
|
|
365
|
+
);
|
|
366
|
+
|
|
367
|
+
canvas = stepResult.canvas;
|
|
368
|
+
previousArgmaxCanvas = stepResult.argmaxCanvas;
|
|
369
|
+
stabilityCounts = stability.counts;
|
|
370
|
+
lastStep = {
|
|
371
|
+
...stepResult,
|
|
372
|
+
step,
|
|
373
|
+
temperature,
|
|
374
|
+
stability,
|
|
375
|
+
};
|
|
376
|
+
|
|
377
|
+
if (stability.allStable && stepResult.meanEntropy < config.confidenceThreshold) {
|
|
378
|
+
break;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
completed = true;
|
|
383
|
+
return {
|
|
384
|
+
canvas,
|
|
385
|
+
argmaxCanvas: previousArgmaxCanvas,
|
|
386
|
+
selfConditioningLogits,
|
|
387
|
+
lastStep,
|
|
388
|
+
stepsRun,
|
|
389
|
+
};
|
|
390
|
+
} finally {
|
|
391
|
+
if (!completed && selfConditioningLogits !== initialSelfConditioningLogits) {
|
|
392
|
+
releaseSelfConditioningState(selfConditioningLogits);
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
}
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
12
|
import type { Tensor } from '../../../../gpu/tensor.js';
|
|
13
|
+
import type { CommandRecorder } from '../../../../gpu/command-recorder.js';
|
|
13
14
|
import type { AttentionState } from './types.js';
|
|
14
15
|
|
|
15
16
|
/**
|
|
@@ -47,6 +48,10 @@ export interface KVCacheDispatchState {
|
|
|
47
48
|
bdpaBasisCount: number;
|
|
48
49
|
hasCache: boolean;
|
|
49
50
|
totalSeqLen: number;
|
|
51
|
+
diffusionGemmaDecoder?: boolean;
|
|
52
|
+
ownedBuffers?: GPUBuffer[] | null;
|
|
53
|
+
encoderSeqLen?: number;
|
|
54
|
+
encoderWindow?: number;
|
|
50
55
|
}
|
|
51
56
|
|
|
52
57
|
/**
|
|
@@ -108,12 +113,31 @@ export function isSlidingLayerType(layerType: string | undefined): boolean;
|
|
|
108
113
|
export function resolveKVCacheState(
|
|
109
114
|
state: AttentionState,
|
|
110
115
|
layerIdx: number,
|
|
111
|
-
kTensor: Tensor,
|
|
112
|
-
vTensor: Tensor,
|
|
116
|
+
kTensor: Tensor | null,
|
|
117
|
+
vTensor: Tensor | null,
|
|
113
118
|
currentSeqLen: number,
|
|
114
119
|
numTokens: number
|
|
115
120
|
): KVCacheDispatchState;
|
|
116
121
|
|
|
122
|
+
export interface DiffusionGemmaDecoderKVStateInput {
|
|
123
|
+
state: AttentionState;
|
|
124
|
+
layerIdx: number;
|
|
125
|
+
kTensor: Tensor;
|
|
126
|
+
vTensor: Tensor;
|
|
127
|
+
currentSeqLen: number;
|
|
128
|
+
numTokens: number;
|
|
129
|
+
numKVHeads: number;
|
|
130
|
+
headDim: number;
|
|
131
|
+
layerType?: string | null;
|
|
132
|
+
slidingWindow?: number | null;
|
|
133
|
+
kvDtype?: string | null;
|
|
134
|
+
recorder?: CommandRecorder | null;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export function createDiffusionGemmaDecoderKVState(
|
|
138
|
+
args: DiffusionGemmaDecoderKVStateInput
|
|
139
|
+
): Promise<KVCacheDispatchState>;
|
|
140
|
+
|
|
117
141
|
/**
|
|
118
142
|
* Build attention dispatch parameters from the KV cache state and layer config.
|
|
119
143
|
*
|
|
@@ -124,8 +148,8 @@ export function resolveKVCacheState(
|
|
|
124
148
|
export function buildAttentionDispatchParams(
|
|
125
149
|
config: DispatchParamsConfig,
|
|
126
150
|
state: AttentionState,
|
|
127
|
-
kTensor: Tensor,
|
|
128
|
-
vTensor: Tensor,
|
|
151
|
+
kTensor: Tensor | null,
|
|
152
|
+
vTensor: Tensor | null,
|
|
129
153
|
kvState: KVCacheDispatchState
|
|
130
154
|
): AttentionDispatchParams;
|
|
131
155
|
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
4
4
|
import { createTensor } from '../../../../gpu/tensor.js';
|
|
5
5
|
import { SlidingWindowKVCache } from '../../../kv-cache.js';
|
|
6
|
+
import { getDevice } from '../../../../gpu/device.js';
|
|
7
|
+
import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
|
|
6
8
|
|
|
7
9
|
// ============================================================================
|
|
8
10
|
// Layer Type Helpers
|
|
@@ -141,6 +143,173 @@ export function resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSe
|
|
|
141
143
|
return kvState;
|
|
142
144
|
}
|
|
143
145
|
|
|
146
|
+
function resolveDecoderEncoderWindow(encoderSeqLen, slidingWindow, layerType) {
|
|
147
|
+
if (!isSlidingLayerType(layerType)) {
|
|
148
|
+
return encoderSeqLen;
|
|
149
|
+
}
|
|
150
|
+
if (!Number.isFinite(slidingWindow) || slidingWindow <= 1) {
|
|
151
|
+
return encoderSeqLen;
|
|
152
|
+
}
|
|
153
|
+
return Math.min(encoderSeqLen, Math.max(0, Math.trunc(slidingWindow) - 1));
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function resolveDecoderKVCacheBuffers(state, layerIdx) {
|
|
157
|
+
if (!state.kvCache?.hasGPUCache?.()) {
|
|
158
|
+
throw new Error(
|
|
159
|
+
`DiffusionGemma decoder attention at layer ${layerIdx} requires an initialized encoder KV cache.`
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
const gpuBuffers = state.kvCache.getGPUBuffers(layerIdx);
|
|
163
|
+
const layout = gpuBuffers?.layout ?? state.kvCache?.layout ?? 'contiguous';
|
|
164
|
+
if (layout !== 'contiguous' && layout !== undefined && layout !== null) {
|
|
165
|
+
throw new Error(
|
|
166
|
+
`DiffusionGemma decoder attention requires contiguous encoder KV cache at layer ${layerIdx}; got "${layout}".`
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
if (!gpuBuffers?.keysGPU || !gpuBuffers?.valuesGPU) {
|
|
170
|
+
throw new Error(
|
|
171
|
+
`DiffusionGemma decoder attention missing contiguous GPU KV buffers at layer ${layerIdx}.`
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
if (!Number.isFinite(gpuBuffers.seqLen) || gpuBuffers.seqLen < 0) {
|
|
175
|
+
throw new Error(
|
|
176
|
+
`DiffusionGemma decoder attention received invalid encoder KV length at layer ${layerIdx}: ${String(gpuBuffers.seqLen)}.`
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
return gpuBuffers;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function copyKVRange(encoder, source, sourceOffset, target, targetOffset, size) {
|
|
183
|
+
if (size <= 0) return;
|
|
184
|
+
encoder.copyBufferToBuffer(source, sourceOffset, target, targetOffset, size);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export async function createDiffusionGemmaDecoderKVState({
|
|
188
|
+
state,
|
|
189
|
+
layerIdx,
|
|
190
|
+
kTensor,
|
|
191
|
+
vTensor,
|
|
192
|
+
currentSeqLen,
|
|
193
|
+
numTokens,
|
|
194
|
+
numKVHeads,
|
|
195
|
+
headDim,
|
|
196
|
+
layerType,
|
|
197
|
+
slidingWindow,
|
|
198
|
+
kvDtype,
|
|
199
|
+
recorder = null,
|
|
200
|
+
}) {
|
|
201
|
+
const gpuBuffers = resolveDecoderKVCacheBuffers(state, layerIdx);
|
|
202
|
+
const encoderSeqLen = Math.trunc(gpuBuffers.seqLen);
|
|
203
|
+
if (encoderSeqLen !== currentSeqLen) {
|
|
204
|
+
throw new Error(
|
|
205
|
+
`DiffusionGemma decoder attention expected currentSeqLen=${encoderSeqLen} from encoder KV cache, ` +
|
|
206
|
+
`got ${currentSeqLen}.`
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const encoderWindow = resolveDecoderEncoderWindow(encoderSeqLen, slidingWindow, layerType);
|
|
211
|
+
const concatSeqLen = encoderWindow + numTokens;
|
|
212
|
+
if (concatSeqLen <= 0) {
|
|
213
|
+
throw new Error(`DiffusionGemma decoder attention has empty KV span at layer ${layerIdx}.`);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
const dtype = kvDtype ?? kTensor.dtype;
|
|
217
|
+
const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
|
|
218
|
+
const rowBytes = numKVHeads * headDim * bytesPerElement;
|
|
219
|
+
const encoderStart = encoderSeqLen - encoderWindow;
|
|
220
|
+
const encoderBytes = encoderWindow * rowBytes;
|
|
221
|
+
const canvasBytes = numTokens * rowBytes;
|
|
222
|
+
const concatBytes = concatSeqLen * rowBytes;
|
|
223
|
+
let keysGPU = null;
|
|
224
|
+
let valuesGPU = null;
|
|
225
|
+
try {
|
|
226
|
+
keysGPU = acquireBuffer(concatBytes, undefined, 'diffusion_gemma_decoder_keys');
|
|
227
|
+
valuesGPU = acquireBuffer(concatBytes, undefined, 'diffusion_gemma_decoder_values');
|
|
228
|
+
const device = recorder?.device ?? getDevice();
|
|
229
|
+
if (!device) {
|
|
230
|
+
throw new Error('DiffusionGemma decoder attention requires a GPU device.');
|
|
231
|
+
}
|
|
232
|
+
const encoder = recorder ? recorder.getEncoder() : device.createCommandEncoder();
|
|
233
|
+
copyKVRange(
|
|
234
|
+
encoder,
|
|
235
|
+
gpuBuffers.keysGPU,
|
|
236
|
+
encoderStart * rowBytes,
|
|
237
|
+
keysGPU,
|
|
238
|
+
0,
|
|
239
|
+
encoderBytes
|
|
240
|
+
);
|
|
241
|
+
copyKVRange(
|
|
242
|
+
encoder,
|
|
243
|
+
gpuBuffers.valuesGPU,
|
|
244
|
+
encoderStart * rowBytes,
|
|
245
|
+
valuesGPU,
|
|
246
|
+
0,
|
|
247
|
+
encoderBytes
|
|
248
|
+
);
|
|
249
|
+
copyKVRange(
|
|
250
|
+
encoder,
|
|
251
|
+
kTensor.buffer,
|
|
252
|
+
0,
|
|
253
|
+
keysGPU,
|
|
254
|
+
encoderBytes,
|
|
255
|
+
canvasBytes
|
|
256
|
+
);
|
|
257
|
+
copyKVRange(
|
|
258
|
+
encoder,
|
|
259
|
+
vTensor.buffer,
|
|
260
|
+
0,
|
|
261
|
+
valuesGPU,
|
|
262
|
+
encoderBytes,
|
|
263
|
+
canvasBytes
|
|
264
|
+
);
|
|
265
|
+
if (!recorder) {
|
|
266
|
+
device.queue.submit([encoder.finish()]);
|
|
267
|
+
}
|
|
268
|
+
} catch (error) {
|
|
269
|
+
if (keysGPU) releaseBuffer(keysGPU);
|
|
270
|
+
if (valuesGPU && valuesGPU !== keysGPU) releaseBuffer(valuesGPU);
|
|
271
|
+
throw error;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
return {
|
|
275
|
+
cachedK: keysGPU,
|
|
276
|
+
cachedV: valuesGPU,
|
|
277
|
+
kvLenForAttention: concatSeqLen,
|
|
278
|
+
causalForAttention: false,
|
|
279
|
+
startPosForMask: encoderSeqLen,
|
|
280
|
+
kvStart: 0,
|
|
281
|
+
kvLayout: 'contiguous',
|
|
282
|
+
kvPageTable: null,
|
|
283
|
+
kvPageSize: 0,
|
|
284
|
+
cachedKHot: undefined,
|
|
285
|
+
cachedVHot: undefined,
|
|
286
|
+
cachedKCold: undefined,
|
|
287
|
+
cachedVCold: undefined,
|
|
288
|
+
coldScalesK: null,
|
|
289
|
+
coldScalesV: null,
|
|
290
|
+
coldPackedStride: 0,
|
|
291
|
+
coldQuantMode: 'none',
|
|
292
|
+
coldLen: 0,
|
|
293
|
+
hotLen: 0,
|
|
294
|
+
hotStart: 0,
|
|
295
|
+
hotWindow: 0,
|
|
296
|
+
coldPageTable: null,
|
|
297
|
+
coldPageSize: 0,
|
|
298
|
+
bdpaBasisK: null,
|
|
299
|
+
bdpaBasisV: null,
|
|
300
|
+
bdpaPagedK: null,
|
|
301
|
+
bdpaPagedV: null,
|
|
302
|
+
bdpaIndex: null,
|
|
303
|
+
bdpaBasisCount: 0,
|
|
304
|
+
hasCache: true,
|
|
305
|
+
totalSeqLen: concatSeqLen,
|
|
306
|
+
diffusionGemmaDecoder: true,
|
|
307
|
+
ownedBuffers: [keysGPU, valuesGPU],
|
|
308
|
+
encoderSeqLen,
|
|
309
|
+
encoderWindow,
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
|
|
144
313
|
// ============================================================================
|
|
145
314
|
// Dispatch Parameter Construction
|
|
146
315
|
// ============================================================================
|
|
@@ -221,13 +390,18 @@ export function buildAttentionDispatchParams(config, state, kTensor, vTensor, kv
|
|
|
221
390
|
const attnScale = queryPreAttnScalar ? 1.0 / Math.sqrt(queryPreAttnScalar) : 1.0 / Math.sqrt(headDim);
|
|
222
391
|
|
|
223
392
|
// Cached K/V dtypes
|
|
393
|
+
const kFallbackDtype = kTensor?.dtype ?? resolvedKvCacheDtype;
|
|
394
|
+
const vFallbackDtype = vTensor?.dtype ?? resolvedKvCacheDtype;
|
|
395
|
+
if (!kFallbackDtype || !vFallbackDtype) {
|
|
396
|
+
throw new Error('Attention dispatch requires K/V tensor dtypes or an explicit KV cache dtype.');
|
|
397
|
+
}
|
|
224
398
|
const cachedKDtype = selectRuleValue('inference', 'dtype', 'f16OrFallback', {
|
|
225
399
|
kvDtype: resolvedKvCacheDtype,
|
|
226
|
-
fallback:
|
|
400
|
+
fallback: kFallbackDtype,
|
|
227
401
|
});
|
|
228
402
|
const cachedVDtype = selectRuleValue('inference', 'dtype', 'f16OrFallback', {
|
|
229
403
|
kvDtype: resolvedKvCacheDtype,
|
|
230
|
-
fallback:
|
|
404
|
+
fallback: vFallbackDtype,
|
|
231
405
|
});
|
|
232
406
|
|
|
233
407
|
// Cached K/V tensors (null for tiered, contiguousQuant, and prefill-fallback paths)
|