doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -13,11 +13,19 @@ export async function withMappedReadBuffer(buffer, read) {
|
|
|
13
13
|
|
|
14
14
|
export async function withMappedReadBuffers(entries, read) {
|
|
15
15
|
const mappedEntries = [];
|
|
16
|
+
let mapError = null;
|
|
16
17
|
try {
|
|
17
18
|
await Promise.all(entries.map(async (entry) => {
|
|
18
|
-
|
|
19
|
-
|
|
19
|
+
try {
|
|
20
|
+
await entry.buffer.mapAsync(GPUMapMode.READ);
|
|
21
|
+
mappedEntries.push(entry);
|
|
22
|
+
} catch (error) {
|
|
23
|
+
mapError ??= error;
|
|
24
|
+
}
|
|
20
25
|
}));
|
|
26
|
+
if (mapError) {
|
|
27
|
+
throw mapError;
|
|
28
|
+
}
|
|
21
29
|
const ranges = entries.map((entry) => {
|
|
22
30
|
if (entry.offset != null || entry.size != null) {
|
|
23
31
|
return entry.buffer.getMappedRange(
|
|
@@ -13,11 +13,13 @@ interface UniformCacheEntry {
|
|
|
13
13
|
refCount: number;
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
-
interface UniformCacheStats {
|
|
16
|
+
export interface UniformCacheStats {
|
|
17
17
|
hits: number;
|
|
18
18
|
misses: number;
|
|
19
19
|
evictions: number;
|
|
20
20
|
currentSize: number;
|
|
21
|
+
hitRate?: string;
|
|
22
|
+
pendingDestruction?: number;
|
|
21
23
|
}
|
|
22
24
|
|
|
23
25
|
/**
|
|
@@ -80,6 +82,8 @@ export declare class UniformBufferCache {
|
|
|
80
82
|
getStats(): UniformCacheStats & { hitRate: string; pendingDestruction: number };
|
|
81
83
|
}
|
|
82
84
|
|
|
85
|
+
export function toUniformArrayBuffer(data: ArrayBuffer | ArrayBufferView): ArrayBuffer;
|
|
86
|
+
|
|
83
87
|
/**
|
|
84
88
|
* Release or destroy a uniform buffer appropriately.
|
|
85
89
|
* If the buffer is cached, releases it back to the cache.
|
|
@@ -92,6 +96,11 @@ export function releaseUniformBuffer(buffer: GPUBuffer): void;
|
|
|
92
96
|
*/
|
|
93
97
|
export function getUniformCache(): UniformBufferCache;
|
|
94
98
|
|
|
99
|
+
/**
|
|
100
|
+
* Get stats for the current global uniform cache without creating one.
|
|
101
|
+
*/
|
|
102
|
+
export function getUniformCacheStats(): (UniformCacheStats & { hitRate: string; pendingDestruction: number }) | null;
|
|
103
|
+
|
|
95
104
|
/**
|
|
96
105
|
* Reset the global uniform cache (useful for testing or device loss)
|
|
97
106
|
*/
|
package/src/gpu/uniform-cache.js
CHANGED
|
@@ -4,8 +4,7 @@ import { getDevice, getDeviceEpoch } from './device.js';
|
|
|
4
4
|
import { getRuntimeConfig } from '../config/runtime.js';
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
function
|
|
8
|
-
const view = new Uint8Array(data);
|
|
7
|
+
function hashUniformBytes(view) {
|
|
9
8
|
let hash = 2166136261; // FNV offset basis
|
|
10
9
|
|
|
11
10
|
for (let i = 0; i < view.length; i++) {
|
|
@@ -17,8 +16,21 @@ function hashArrayBuffer(data) {
|
|
|
17
16
|
return (hash >>> 0).toString(16).padStart(8, '0');
|
|
18
17
|
}
|
|
19
18
|
|
|
20
|
-
function copyUniformBytes(
|
|
21
|
-
return new Uint8Array(
|
|
19
|
+
function copyUniformBytes(view) {
|
|
20
|
+
return new Uint8Array(view);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function toUniformArrayBuffer(data) {
|
|
24
|
+
if (data instanceof ArrayBuffer) {
|
|
25
|
+
return data;
|
|
26
|
+
}
|
|
27
|
+
if (ArrayBuffer.isView(data)) {
|
|
28
|
+
if (data.byteOffset === 0 && data.byteLength === data.buffer.byteLength) {
|
|
29
|
+
return data.buffer;
|
|
30
|
+
}
|
|
31
|
+
return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
|
|
32
|
+
}
|
|
33
|
+
throw new Error('Uniform data must be ArrayBuffer or ArrayBufferView.');
|
|
22
34
|
}
|
|
23
35
|
|
|
24
36
|
function equalUniformBytes(a, b) {
|
|
@@ -62,14 +74,14 @@ export class UniformBufferCache {
|
|
|
62
74
|
|
|
63
75
|
|
|
64
76
|
getOrCreate(data, label) {
|
|
65
|
-
const
|
|
66
|
-
const
|
|
77
|
+
const dataView = new Uint8Array(data);
|
|
78
|
+
const baseKey = `${data.byteLength}:${hashUniformBytes(dataView)}`;
|
|
67
79
|
let key = baseKey;
|
|
68
80
|
let suffix = 0;
|
|
69
81
|
let existing = this.#cache.get(key);
|
|
70
82
|
|
|
71
83
|
while (existing) {
|
|
72
|
-
if (equalUniformBytes(existing.bytes,
|
|
84
|
+
if (equalUniformBytes(existing.bytes, dataView)) {
|
|
73
85
|
existing.lastUsed = performance.now();
|
|
74
86
|
existing.refCount++;
|
|
75
87
|
this.#stats.hits++;
|
|
@@ -80,6 +92,8 @@ export class UniformBufferCache {
|
|
|
80
92
|
existing = this.#cache.get(key);
|
|
81
93
|
}
|
|
82
94
|
|
|
95
|
+
const dataBytes = copyUniformBytes(dataView);
|
|
96
|
+
|
|
83
97
|
// Cache miss - create new buffer
|
|
84
98
|
this.#stats.misses++;
|
|
85
99
|
|
|
@@ -230,6 +244,10 @@ export function getUniformCache() {
|
|
|
230
244
|
return globalUniformCache;
|
|
231
245
|
}
|
|
232
246
|
|
|
247
|
+
export function getUniformCacheStats() {
|
|
248
|
+
return globalUniformCache ? globalUniformCache.getStats() : null;
|
|
249
|
+
}
|
|
250
|
+
|
|
233
251
|
|
|
234
252
|
export function resetUniformCache() {
|
|
235
253
|
if (globalUniformCache) {
|
|
@@ -10,8 +10,32 @@
|
|
|
10
10
|
* Use WeightBuffer for model weights (static, may be quantized).
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
export type WeightDtype = 'f16' | 'f32' | 'bf16' | 'q4k' | 'q8';
|
|
13
|
+
export type WeightDtype = 'f16' | 'f32' | 'bf16' | 'q4k' | 'q8' | 'litert_int4' | 'w4a16';
|
|
14
14
|
export type WeightLayout = 'row' | 'column';
|
|
15
|
+
export type WeightStorageEncoding = 'signed' | 'offset_binary';
|
|
16
|
+
export type WeightScaleDtype = 'f16' | 'bf16' | 'f32';
|
|
17
|
+
|
|
18
|
+
export interface WeightMetadata {
|
|
19
|
+
readonly storageEncoding?: WeightStorageEncoding;
|
|
20
|
+
readonly scaleBuffer?: GPUBuffer;
|
|
21
|
+
readonly scaleDtype?: WeightScaleDtype;
|
|
22
|
+
readonly groupsPerRow?: number;
|
|
23
|
+
readonly splitGatherSectionCount?: number;
|
|
24
|
+
readonly sourceKernel?: {
|
|
25
|
+
readonly kernel?: string | null;
|
|
26
|
+
readonly entry?: string | null;
|
|
27
|
+
readonly digest?: string | null;
|
|
28
|
+
};
|
|
29
|
+
readonly storageType?: 'functional_descriptor' | string;
|
|
30
|
+
readonly descriptorHash?: string;
|
|
31
|
+
readonly descriptorBytes?: number;
|
|
32
|
+
readonly denseF16Bytes?: number;
|
|
33
|
+
readonly compressionRatio?: number | null;
|
|
34
|
+
readonly proofStatus?: string | null;
|
|
35
|
+
readonly proofStatusGate?: Record<string, string> | null;
|
|
36
|
+
readonly descriptorShape?: readonly [number, number];
|
|
37
|
+
readonly cropShape?: readonly [number, number];
|
|
38
|
+
}
|
|
15
39
|
|
|
16
40
|
export interface CpuTensorRangeSource {
|
|
17
41
|
readonly kind: 'tensor_range_source';
|
|
@@ -29,6 +53,8 @@ export interface CpuWeightBuffer {
|
|
|
29
53
|
readonly layout: WeightLayout;
|
|
30
54
|
readonly shape: readonly number[];
|
|
31
55
|
readonly label?: string;
|
|
56
|
+
gpuSplitWeight?: SplitWeightBuffer | null;
|
|
57
|
+
readonly metadata?: WeightMetadata;
|
|
32
58
|
}
|
|
33
59
|
|
|
34
60
|
/**
|
|
@@ -45,6 +71,26 @@ export interface WeightBuffer {
|
|
|
45
71
|
readonly buffer: GPUBuffer;
|
|
46
72
|
readonly layout: WeightLayout;
|
|
47
73
|
}>>>;
|
|
74
|
+
readonly metadata?: WeightMetadata;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export interface SplitWeightSection {
|
|
78
|
+
readonly buffer: GPUBuffer;
|
|
79
|
+
readonly rowStart: number;
|
|
80
|
+
readonly rowCount: number;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* GPU-resident weight whose row dimension is split across multiple buffers.
|
|
85
|
+
*/
|
|
86
|
+
export interface SplitWeightBuffer {
|
|
87
|
+
readonly kind: 'split_weight_buffer';
|
|
88
|
+
readonly sections: readonly SplitWeightSection[];
|
|
89
|
+
readonly dtype: WeightDtype;
|
|
90
|
+
readonly layout: WeightLayout;
|
|
91
|
+
readonly shape: readonly number[];
|
|
92
|
+
readonly label?: string;
|
|
93
|
+
readonly metadata?: WeightMetadata;
|
|
48
94
|
}
|
|
49
95
|
|
|
50
96
|
/**
|
|
@@ -87,7 +133,8 @@ export function createWeightBuffer(
|
|
|
87
133
|
materializations?: Partial<Record<WeightDtype, {
|
|
88
134
|
buffer: GPUBuffer;
|
|
89
135
|
layout?: WeightLayout;
|
|
90
|
-
}>> | null
|
|
136
|
+
}>> | null,
|
|
137
|
+
metadata?: WeightMetadata | null
|
|
91
138
|
): WeightBuffer;
|
|
92
139
|
|
|
93
140
|
/**
|
|
@@ -101,6 +148,18 @@ export function createCpuWeightBuffer(
|
|
|
101
148
|
label?: string
|
|
102
149
|
): CpuWeightBuffer;
|
|
103
150
|
|
|
151
|
+
/**
|
|
152
|
+
* Create a GPU split weight buffer with explicit row-section metadata.
|
|
153
|
+
*/
|
|
154
|
+
export function createSplitWeightBuffer(
|
|
155
|
+
sections: SplitWeightSection[],
|
|
156
|
+
dtype: WeightDtype,
|
|
157
|
+
layout: WeightLayout,
|
|
158
|
+
shape: number[],
|
|
159
|
+
label?: string,
|
|
160
|
+
metadata?: WeightMetadata | null
|
|
161
|
+
): SplitWeightBuffer;
|
|
162
|
+
|
|
104
163
|
/**
|
|
105
164
|
* Check if weight is stored in column-major (pre-transposed) format.
|
|
106
165
|
* Column-major weights use transposeB=false in matmul.
|
|
@@ -117,28 +176,38 @@ export function isWeightBuffer(value: unknown): value is WeightBuffer;
|
|
|
117
176
|
*/
|
|
118
177
|
export function isCpuWeightBuffer(value: unknown): value is CpuWeightBuffer;
|
|
119
178
|
|
|
179
|
+
/**
|
|
180
|
+
* Check if value is a GPU-resident split weight buffer.
|
|
181
|
+
*/
|
|
182
|
+
export function isSplitWeightBuffer(value: unknown): value is SplitWeightBuffer;
|
|
183
|
+
|
|
120
184
|
/**
|
|
121
185
|
* Extract the raw GPUBuffer from either a WeightBuffer or raw GPUBuffer.
|
|
122
186
|
* Used for backwards compatibility during migration.
|
|
123
187
|
*/
|
|
124
|
-
export function getBuffer(weight: GPUBuffer | WeightBuffer | TensorLike): GPUBuffer;
|
|
188
|
+
export function getBuffer(weight: GPUBuffer | WeightBuffer | TensorLike | SplitWeightBuffer): GPUBuffer | SplitWeightBuffer;
|
|
125
189
|
|
|
126
190
|
/**
|
|
127
191
|
* Get layout from WeightBuffer, or null for raw GPUBuffer.
|
|
128
192
|
* Used for auto-resolving transposeB in matmul.
|
|
129
193
|
*/
|
|
130
|
-
export function getLayout(weight: GPUBuffer | WeightBuffer | TensorLike): WeightLayout | null;
|
|
194
|
+
export function getLayout(weight: GPUBuffer | WeightBuffer | TensorLike | SplitWeightBuffer): WeightLayout | null;
|
|
131
195
|
|
|
132
196
|
/**
|
|
133
197
|
* Get dtype from WeightBuffer, tagged raw GPUBuffer, or TensorLike.
|
|
134
198
|
*/
|
|
135
|
-
export function getWeightDtype(weight: GPUBuffer | WeightBuffer | TensorLike): WeightDtype | TensorLike['dtype'] | null;
|
|
199
|
+
export function getWeightDtype(weight: GPUBuffer | WeightBuffer | TensorLike | SplitWeightBuffer): WeightDtype | TensorLike['dtype'] | null;
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Get optional quantized-weight metadata from a weight wrapper.
|
|
203
|
+
*/
|
|
204
|
+
export function getWeightMetadata(weight: unknown): WeightMetadata | null;
|
|
136
205
|
|
|
137
206
|
/**
|
|
138
207
|
* Resolve a preferred materialization view from a WeightBuffer when alternate
|
|
139
208
|
* dense/quantized buffers are available.
|
|
140
209
|
*/
|
|
141
210
|
export function resolveWeightBufferMaterialization(
|
|
142
|
-
weight: GPUBuffer | WeightBuffer | TensorLike,
|
|
211
|
+
weight: GPUBuffer | WeightBuffer | TensorLike | SplitWeightBuffer,
|
|
143
212
|
preferredDtype?: WeightDtype | TensorLike['dtype'] | null
|
|
144
|
-
): GPUBuffer | WeightBuffer | TensorLike;
|
|
213
|
+
): GPUBuffer | WeightBuffer | TensorLike | SplitWeightBuffer;
|
package/src/gpu/weight-buffer.js
CHANGED
|
@@ -46,6 +46,22 @@ function normalizeDtype(dtype) {
|
|
|
46
46
|
return value.length > 0 ? value : null;
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
function normalizeWeightMetadata(metadata) {
|
|
50
|
+
if (!metadata || typeof metadata !== 'object') {
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
const normalized = {};
|
|
54
|
+
for (const [key, value] of Object.entries(metadata)) {
|
|
55
|
+
if (value === undefined) {
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
normalized[key] = key === 'storageEncoding' && typeof value === 'string'
|
|
59
|
+
? value.toLowerCase()
|
|
60
|
+
: value;
|
|
61
|
+
}
|
|
62
|
+
return Object.keys(normalized).length > 0 ? Object.freeze(normalized) : null;
|
|
63
|
+
}
|
|
64
|
+
|
|
49
65
|
export function tagBufferDtype(buffer, dtype) {
|
|
50
66
|
if (!canTrackBuffer(buffer)) return;
|
|
51
67
|
const normalized = normalizeDtype(dtype);
|
|
@@ -65,9 +81,11 @@ export function createWeightBuffer(
|
|
|
65
81
|
layout,
|
|
66
82
|
shape,
|
|
67
83
|
label,
|
|
68
|
-
materializations = null
|
|
84
|
+
materializations = null,
|
|
85
|
+
metadata = null
|
|
69
86
|
) {
|
|
70
87
|
tagBufferDtype(buffer, dtype);
|
|
88
|
+
const normalizedMetadata = normalizeWeightMetadata(metadata);
|
|
71
89
|
const normalizedMaterializations = {};
|
|
72
90
|
if (materializations && typeof materializations === 'object') {
|
|
73
91
|
for (const [materializationDtype, descriptor] of Object.entries(materializations)) {
|
|
@@ -92,6 +110,7 @@ export function createWeightBuffer(
|
|
|
92
110
|
shape: Object.freeze([...shape]),
|
|
93
111
|
label,
|
|
94
112
|
materializations: Object.freeze(normalizedMaterializations),
|
|
113
|
+
...(normalizedMetadata ? { metadata: normalizedMetadata } : {}),
|
|
95
114
|
};
|
|
96
115
|
}
|
|
97
116
|
|
|
@@ -112,6 +131,46 @@ export function createCpuWeightBuffer(
|
|
|
112
131
|
};
|
|
113
132
|
}
|
|
114
133
|
|
|
134
|
+
export function createSplitWeightBuffer(
|
|
135
|
+
sections,
|
|
136
|
+
dtype,
|
|
137
|
+
layout,
|
|
138
|
+
shape,
|
|
139
|
+
label,
|
|
140
|
+
metadata = null
|
|
141
|
+
) {
|
|
142
|
+
if (!Array.isArray(sections) || sections.length === 0) {
|
|
143
|
+
throw new Error('[WeightBuffer] split weight buffer requires at least one section.');
|
|
144
|
+
}
|
|
145
|
+
const normalizedSections = sections.map((section, index) => {
|
|
146
|
+
if (!section || !canTrackBuffer(section.buffer)) {
|
|
147
|
+
throw new Error(`[WeightBuffer] split section ${index} is missing a GPU buffer.`);
|
|
148
|
+
}
|
|
149
|
+
if (!Number.isInteger(section.rowStart) || section.rowStart < 0) {
|
|
150
|
+
throw new Error(`[WeightBuffer] split section ${index} has invalid rowStart.`);
|
|
151
|
+
}
|
|
152
|
+
if (!Number.isInteger(section.rowCount) || section.rowCount <= 0) {
|
|
153
|
+
throw new Error(`[WeightBuffer] split section ${index} has invalid rowCount.`);
|
|
154
|
+
}
|
|
155
|
+
tagBufferDtype(section.buffer, dtype);
|
|
156
|
+
return Object.freeze({
|
|
157
|
+
buffer: section.buffer,
|
|
158
|
+
rowStart: section.rowStart,
|
|
159
|
+
rowCount: section.rowCount,
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
const normalizedMetadata = normalizeWeightMetadata(metadata);
|
|
163
|
+
return Object.freeze({
|
|
164
|
+
kind: 'split_weight_buffer',
|
|
165
|
+
sections: Object.freeze(normalizedSections),
|
|
166
|
+
dtype,
|
|
167
|
+
layout,
|
|
168
|
+
shape: Object.freeze([...shape]),
|
|
169
|
+
label,
|
|
170
|
+
...(normalizedMetadata ? { metadata: normalizedMetadata } : {}),
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
|
|
115
174
|
|
|
116
175
|
export function isColumnMajor(weight) {
|
|
117
176
|
return weight.layout === 'column';
|
|
@@ -141,6 +200,19 @@ export function isCpuWeightBuffer(value) {
|
|
|
141
200
|
);
|
|
142
201
|
}
|
|
143
202
|
|
|
203
|
+
|
|
204
|
+
export function isSplitWeightBuffer(value) {
|
|
205
|
+
return (
|
|
206
|
+
typeof value === 'object' &&
|
|
207
|
+
value !== null &&
|
|
208
|
+
value.kind === 'split_weight_buffer' &&
|
|
209
|
+
Array.isArray(value.sections) &&
|
|
210
|
+
'dtype' in value &&
|
|
211
|
+
'layout' in value &&
|
|
212
|
+
'shape' in value
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
|
|
144
216
|
// Intentionally lenient: does not call isValidGPUBuffer on value.buffer.
|
|
145
217
|
// This function is used as a structural duck-type check to distinguish tensor-like
|
|
146
218
|
// wrappers from raw GPUBuffer objects. The buffer property may hold a CPU typed array,
|
|
@@ -165,17 +237,29 @@ export function getBuffer(weight) {
|
|
|
165
237
|
|
|
166
238
|
|
|
167
239
|
export function getLayout(weight) {
|
|
240
|
+
if (isSplitWeightBuffer(weight)) return weight.layout;
|
|
168
241
|
return isWeightBuffer(weight) ? weight.layout : null;
|
|
169
242
|
}
|
|
170
243
|
|
|
171
244
|
|
|
172
245
|
export function getWeightDtype(weight) {
|
|
246
|
+
if (isSplitWeightBuffer(weight)) return weight.dtype;
|
|
173
247
|
if (isWeightBuffer(weight)) return weight.dtype;
|
|
174
248
|
if (isTensorLike(weight)) return weight.dtype;
|
|
175
249
|
return getBufferDtype(weight);
|
|
176
250
|
}
|
|
177
251
|
|
|
252
|
+
export function getWeightMetadata(weight) {
|
|
253
|
+
if (isWeightBuffer(weight) || isCpuWeightBuffer(weight) || isSplitWeightBuffer(weight)) {
|
|
254
|
+
return weight.metadata ?? null;
|
|
255
|
+
}
|
|
256
|
+
return null;
|
|
257
|
+
}
|
|
258
|
+
|
|
178
259
|
export function resolveWeightBufferMaterialization(weight, preferredDtype = null) {
|
|
260
|
+
if (isSplitWeightBuffer(weight)) {
|
|
261
|
+
return weight;
|
|
262
|
+
}
|
|
179
263
|
if (!isWeightBuffer(weight) || preferredDtype == null || preferredDtype === weight.dtype) {
|
|
180
264
|
return weight;
|
|
181
265
|
}
|
|
@@ -13,6 +13,261 @@ import {
|
|
|
13
13
|
buildTimingDiagnostics,
|
|
14
14
|
} from './browser-harness-suite-helpers.js';
|
|
15
15
|
|
|
16
|
+
function isDiffusionGemmaManifest(manifest) {
|
|
17
|
+
return manifest?.modelType === 'diffusion_gemma';
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function readOptionalPositiveInteger(source, field, label) {
|
|
21
|
+
const value = source?.[field];
|
|
22
|
+
if (value == null) return null;
|
|
23
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
24
|
+
throw new Error(`${label}.${field} must be a positive integer when provided.`);
|
|
25
|
+
}
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function readOptionalInteger(source, field, label) {
|
|
30
|
+
const value = source?.[field];
|
|
31
|
+
if (value == null) return null;
|
|
32
|
+
if (!Number.isInteger(value)) {
|
|
33
|
+
throw new Error(`${label}.${field} must be an integer when provided.`);
|
|
34
|
+
}
|
|
35
|
+
return value;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function resolveDiffusionGemmaRunConfig(runtimeConfig, manifest) {
|
|
39
|
+
const label = 'runtime.inference.diffusionGemma';
|
|
40
|
+
const source = runtimeConfig?.inference?.diffusionGemma ?? null;
|
|
41
|
+
if (source != null && (typeof source !== 'object' || Array.isArray(source))) {
|
|
42
|
+
throw new Error(`${label} must be an object when provided.`);
|
|
43
|
+
}
|
|
44
|
+
const contract = manifest?.inference?.diffusionGemma ?? null;
|
|
45
|
+
const maxNewTokens = readOptionalPositiveInteger(source, 'maxNewTokens', label)
|
|
46
|
+
?? contract?.maxNewTokens
|
|
47
|
+
?? null;
|
|
48
|
+
const canvasLength = readOptionalPositiveInteger(source, 'canvasLength', label)
|
|
49
|
+
?? contract?.canvasLength
|
|
50
|
+
?? null;
|
|
51
|
+
const maxDenoisingSteps = readOptionalPositiveInteger(source, 'maxDenoisingSteps', label)
|
|
52
|
+
?? contract?.maxDenoisingSteps
|
|
53
|
+
?? null;
|
|
54
|
+
const seed = readOptionalInteger(source, 'seed', label);
|
|
55
|
+
return {
|
|
56
|
+
maxNewTokens,
|
|
57
|
+
canvasLength,
|
|
58
|
+
maxDenoisingSteps,
|
|
59
|
+
seed,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function decodeDiffusionGemmaOutput(pipeline, tokenIds) {
|
|
64
|
+
const ids = Array.from(tokenIds);
|
|
65
|
+
if (typeof pipeline?.tokenizer?.decode === 'function') {
|
|
66
|
+
return pipeline.tokenizer.decode(ids);
|
|
67
|
+
}
|
|
68
|
+
return ids.join(' ');
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function averageRounded(values) {
|
|
72
|
+
if (!values.length) return 0;
|
|
73
|
+
return Math.round(values.reduce((sum, value) => sum + value, 0) / values.length);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
async function runDiffusionGemmaSuite(options, context) {
|
|
77
|
+
const {
|
|
78
|
+
startTime,
|
|
79
|
+
runtimeConfig,
|
|
80
|
+
captureOutput,
|
|
81
|
+
cacheMode,
|
|
82
|
+
loadMode,
|
|
83
|
+
warmupRuns,
|
|
84
|
+
timedRuns,
|
|
85
|
+
harness,
|
|
86
|
+
} = context;
|
|
87
|
+
const prompt = resolvePrompt(runtimeConfig);
|
|
88
|
+
const runConfig = resolveDiffusionGemmaRunConfig(runtimeConfig, harness.manifest);
|
|
89
|
+
const totalMs = [];
|
|
90
|
+
const prefillMs = [];
|
|
91
|
+
const denoiseMs = [];
|
|
92
|
+
const vaeMs = [];
|
|
93
|
+
const prefillTokens = [];
|
|
94
|
+
const decodeTokens = [];
|
|
95
|
+
const denoiseSteps = [];
|
|
96
|
+
const tokensPerForward = [];
|
|
97
|
+
let output = null;
|
|
98
|
+
|
|
99
|
+
for (let i = 0; i < warmupRuns + timedRuns; i += 1) {
|
|
100
|
+
harness.pipeline.reset?.();
|
|
101
|
+
const generateOptions = {};
|
|
102
|
+
if (runConfig.maxNewTokens != null) {
|
|
103
|
+
generateOptions.maxNewTokens = runConfig.maxNewTokens;
|
|
104
|
+
}
|
|
105
|
+
if (runConfig.seed != null) {
|
|
106
|
+
generateOptions.seed = runConfig.seed;
|
|
107
|
+
}
|
|
108
|
+
if (runConfig.canvasLength != null) {
|
|
109
|
+
generateOptions.canvasLength = runConfig.canvasLength;
|
|
110
|
+
}
|
|
111
|
+
if (runConfig.maxDenoisingSteps != null) {
|
|
112
|
+
generateOptions.maxDenoisingSteps = runConfig.maxDenoisingSteps;
|
|
113
|
+
}
|
|
114
|
+
const runStart = performance.now();
|
|
115
|
+
const tokenIds = await harness.pipeline.generateTokenIds(prompt, generateOptions);
|
|
116
|
+
const wallMs = Math.max(0, performance.now() - runStart);
|
|
117
|
+
const stats = harness.pipeline.getStats?.() ?? {};
|
|
118
|
+
|
|
119
|
+
if (captureOutput && i === warmupRuns + timedRuns - 1) {
|
|
120
|
+
output = {
|
|
121
|
+
tokenIds: Array.from(tokenIds),
|
|
122
|
+
text: decodeDiffusionGemmaOutput(harness.pipeline, tokenIds),
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (i < warmupRuns) continue;
|
|
127
|
+
|
|
128
|
+
totalMs.push(Number.isFinite(stats.totalTimeMs) ? stats.totalTimeMs : wallMs);
|
|
129
|
+
prefillMs.push(Number.isFinite(stats.prefillTimeMs) ? stats.prefillTimeMs : 0);
|
|
130
|
+
denoiseMs.push(Number.isFinite(stats.decodeTimeMs) ? stats.decodeTimeMs : wallMs);
|
|
131
|
+
vaeMs.push(0);
|
|
132
|
+
prefillTokens.push(Number.isFinite(stats.prefillTokens) ? stats.prefillTokens : 0);
|
|
133
|
+
decodeTokens.push(Number.isFinite(stats.decodeTokens) ? stats.decodeTokens : tokenIds.length);
|
|
134
|
+
denoiseSteps.push(Number.isFinite(stats.denoiseSteps) ? stats.denoiseSteps : 0);
|
|
135
|
+
if (Number.isFinite(stats.tokensPerForward)) {
|
|
136
|
+
tokensPerForward.push(stats.tokensPerForward);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const memoryStats = typeof harness.pipeline?.getMemoryStats === 'function'
|
|
141
|
+
? harness.pipeline.getMemoryStats()
|
|
142
|
+
: null;
|
|
143
|
+
|
|
144
|
+
if (typeof harness.pipeline.unload === 'function' && !options.keepPipeline) {
|
|
145
|
+
await harness.pipeline.unload();
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const results = [
|
|
149
|
+
{
|
|
150
|
+
name: 'diffusion_gemma',
|
|
151
|
+
passed: totalMs.length > 0 && decodeTokens.some((value) => value > 0),
|
|
152
|
+
duration: totalMs.reduce((sum, value) => sum + value, 0),
|
|
153
|
+
error: totalMs.length === 0
|
|
154
|
+
? 'No DiffusionGemma runs completed'
|
|
155
|
+
: (decodeTokens.some((value) => value > 0) ? undefined : 'No DiffusionGemma tokens generated'),
|
|
156
|
+
},
|
|
157
|
+
];
|
|
158
|
+
|
|
159
|
+
const summary = buildSuiteSummary('diffusion', results, startTime);
|
|
160
|
+
const cpuStats = {
|
|
161
|
+
totalMs: computeSampleStats(totalMs),
|
|
162
|
+
prefillMs: computeSampleStats(prefillMs),
|
|
163
|
+
denoiseMs: computeSampleStats(denoiseMs),
|
|
164
|
+
vaeMs: computeSampleStats(vaeMs),
|
|
165
|
+
};
|
|
166
|
+
const gpuStats = { available: false };
|
|
167
|
+
const avgPrefillTokens = averageRounded(prefillTokens);
|
|
168
|
+
const avgDecodeTokens = averageRounded(decodeTokens);
|
|
169
|
+
const avgDenoiseSteps = averageRounded(denoiseSteps);
|
|
170
|
+
const basePerformanceArtifact = buildDiffusionPerformanceArtifact({
|
|
171
|
+
warmupRuns,
|
|
172
|
+
timedRuns,
|
|
173
|
+
width: null,
|
|
174
|
+
height: null,
|
|
175
|
+
steps: avgDenoiseSteps,
|
|
176
|
+
guidanceScale: null,
|
|
177
|
+
avgPrefillTokens,
|
|
178
|
+
avgDecodeTokens,
|
|
179
|
+
cpuStats,
|
|
180
|
+
gpuStats,
|
|
181
|
+
modality: 'text',
|
|
182
|
+
});
|
|
183
|
+
const diffusionPerformanceArtifact = {
|
|
184
|
+
...basePerformanceArtifact,
|
|
185
|
+
diffusionGemma: {
|
|
186
|
+
canvasLength: runConfig.canvasLength,
|
|
187
|
+
maxDenoisingSteps: runConfig.maxDenoisingSteps,
|
|
188
|
+
maxNewTokens: runConfig.maxNewTokens,
|
|
189
|
+
tokensPerForward: computeSampleStats(tokensPerForward),
|
|
190
|
+
denoiseSteps: computeSampleStats(denoiseSteps),
|
|
191
|
+
},
|
|
192
|
+
};
|
|
193
|
+
const prefillMsMedian = safeStatsValue(cpuStats.prefillMs?.median);
|
|
194
|
+
const denoiseMsMedian = safeStatsValue(cpuStats.denoiseMs?.median);
|
|
195
|
+
const totalMsMedian = safeStatsValue(cpuStats.totalMs?.median);
|
|
196
|
+
const timing = buildCanonicalTiming({
|
|
197
|
+
modelLoadMs: harness.modelLoadMs ?? 0,
|
|
198
|
+
firstTokenMs: null,
|
|
199
|
+
firstResponseMs: null,
|
|
200
|
+
prefillMs: prefillMsMedian,
|
|
201
|
+
decodeMs: denoiseMsMedian,
|
|
202
|
+
totalRunMs: totalMsMedian,
|
|
203
|
+
prefillTokensPerSec: diffusionPerformanceArtifact.throughput.prefillTokensPerSec,
|
|
204
|
+
decodeTokensPerSec: diffusionPerformanceArtifact.throughput.decodeTokensPerSec,
|
|
205
|
+
cacheMode,
|
|
206
|
+
loadMode,
|
|
207
|
+
});
|
|
208
|
+
const timingDiagnostics = buildTimingDiagnostics(timing, {
|
|
209
|
+
source: 'doppler',
|
|
210
|
+
prefillSemantics: 'diffusion_gemma_encoder_prefill_phase',
|
|
211
|
+
decodeSemantics: 'diffusion_gemma_canvas_denoising_phase',
|
|
212
|
+
});
|
|
213
|
+
const metricsWithContracts = buildSuiteContractMetrics(
|
|
214
|
+
'diffusion',
|
|
215
|
+
{
|
|
216
|
+
warmupRuns,
|
|
217
|
+
timedRuns,
|
|
218
|
+
prompt,
|
|
219
|
+
width: null,
|
|
220
|
+
height: null,
|
|
221
|
+
steps: avgDenoiseSteps,
|
|
222
|
+
guidanceScale: null,
|
|
223
|
+
maxNewTokens: runConfig.maxNewTokens,
|
|
224
|
+
canvasLength: runConfig.canvasLength,
|
|
225
|
+
avgPrefillTokens,
|
|
226
|
+
avgDecodeTokens,
|
|
227
|
+
tokensPerForward: computeSampleStats(tokensPerForward),
|
|
228
|
+
latency: {
|
|
229
|
+
totalMs: cpuStats.totalMs,
|
|
230
|
+
prefillMs: cpuStats.prefillMs,
|
|
231
|
+
denoiseMs: cpuStats.denoiseMs,
|
|
232
|
+
vaeMs: cpuStats.vaeMs,
|
|
233
|
+
},
|
|
234
|
+
throughput: {
|
|
235
|
+
prefillTokensPerSec: diffusionPerformanceArtifact.throughput.prefillTokensPerSec,
|
|
236
|
+
decodeTokensPerSec: diffusionPerformanceArtifact.throughput.decodeTokensPerSec,
|
|
237
|
+
decodeStepsPerSec: diffusionPerformanceArtifact.throughput.decodeStepsPerSec,
|
|
238
|
+
tokensPerForward: diffusionPerformanceArtifact.diffusionGemma.tokensPerForward,
|
|
239
|
+
},
|
|
240
|
+
cpu: cpuStats,
|
|
241
|
+
gpu: gpuStats,
|
|
242
|
+
performanceArtifact: diffusionPerformanceArtifact,
|
|
243
|
+
},
|
|
244
|
+
harness.manifest
|
|
245
|
+
);
|
|
246
|
+
|
|
247
|
+
return {
|
|
248
|
+
...summary,
|
|
249
|
+
modelId: options.modelId || harness.manifest?.modelId || 'unknown',
|
|
250
|
+
cacheMode,
|
|
251
|
+
loadMode,
|
|
252
|
+
env: {
|
|
253
|
+
library: 'doppler',
|
|
254
|
+
runtime: 'browser',
|
|
255
|
+
device: 'webgpu',
|
|
256
|
+
browserUserAgent: typeof navigator !== 'undefined' ? (navigator.userAgent || null) : null,
|
|
257
|
+
browserPlatform: typeof navigator !== 'undefined' ? (navigator.platform || null) : null,
|
|
258
|
+
browserLanguage: typeof navigator !== 'undefined' ? (navigator.language || null) : null,
|
|
259
|
+
browserVendor: typeof navigator !== 'undefined' ? (navigator.vendor || null) : null,
|
|
260
|
+
},
|
|
261
|
+
timing,
|
|
262
|
+
timingDiagnostics,
|
|
263
|
+
output,
|
|
264
|
+
metrics: metricsWithContracts,
|
|
265
|
+
memoryStats,
|
|
266
|
+
deviceInfo: resolveDeviceInfo(),
|
|
267
|
+
pipeline: options.keepPipeline ? harness.pipeline : null,
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
|
|
16
271
|
export async function runDiffusionSuite(options = {}) {
|
|
17
272
|
const startTime = performance.now();
|
|
18
273
|
const runtimeConfig = getRuntimeConfig();
|
|
@@ -22,6 +277,20 @@ export async function runDiffusionSuite(options = {}) {
|
|
|
22
277
|
const benchConfig = runtimeConfig.shared?.benchmark?.run || {};
|
|
23
278
|
const warmupRuns = Math.max(0, Math.floor(benchConfig.warmupRuns ?? 0));
|
|
24
279
|
const timedRuns = Math.max(1, Math.floor(benchConfig.timedRuns ?? 1));
|
|
280
|
+
const harness = await initializeSuiteModel(options);
|
|
281
|
+
|
|
282
|
+
if (isDiffusionGemmaManifest(harness.manifest)) {
|
|
283
|
+
return runDiffusionGemmaSuite(options, {
|
|
284
|
+
startTime,
|
|
285
|
+
runtimeConfig,
|
|
286
|
+
captureOutput,
|
|
287
|
+
cacheMode,
|
|
288
|
+
loadMode,
|
|
289
|
+
warmupRuns,
|
|
290
|
+
timedRuns,
|
|
291
|
+
harness,
|
|
292
|
+
});
|
|
293
|
+
}
|
|
25
294
|
|
|
26
295
|
const diffusionConfig = runtimeConfig.inference?.diffusion;
|
|
27
296
|
if (!diffusionConfig) {
|
|
@@ -50,7 +319,6 @@ export async function runDiffusionSuite(options = {}) {
|
|
|
50
319
|
throw new Error('runtime.inference.diffusion.scheduler.guidanceScale must be set for diffusion harness runs.');
|
|
51
320
|
}
|
|
52
321
|
|
|
53
|
-
const harness = await initializeSuiteModel(options);
|
|
54
322
|
const totalMs = [];
|
|
55
323
|
const prefillMs = [];
|
|
56
324
|
const denoiseMs = [];
|