doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -27,6 +27,23 @@ function selectQ4KFusedVariant(isM1, wantF16Output, aDtype, phase) {
|
|
|
27
27
|
return selectKernelRuleValue('matmul', 'q4kFusedVariant', { useF16A, useF16Out, isM1, isPrefill });
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
function selectLiteRTInt4FusedVariant(outputDtype, aDtype, transposeB) {
|
|
31
|
+
return selectKernelRuleValue('matmul', 'litertInt4FusedVariant', {
|
|
32
|
+
activationDtype: aDtype,
|
|
33
|
+
outputDtype,
|
|
34
|
+
transposeB,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function selectW4A16FusedVariant(outputDtype, aDtype, transposeB, isPrefill) {
|
|
39
|
+
return selectKernelRuleValue('matmul', 'w4a16FusedVariant', {
|
|
40
|
+
activationDtype: aDtype,
|
|
41
|
+
outputDtype,
|
|
42
|
+
transposeB,
|
|
43
|
+
isPrefill,
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
|
|
30
47
|
|
|
31
48
|
export function resolveMatmulPhase(M, phaseOverride = null) {
|
|
32
49
|
if (phaseOverride != null) {
|
|
@@ -207,6 +224,14 @@ export function getMatmulBindingSizes(label, A, B, M, N, K, aDtype, bDtype, tran
|
|
|
207
224
|
const numBlocksPerRow = Math.ceil(K / QK_K);
|
|
208
225
|
bBindingSize = Math.ceil((N * numBlocksPerRow * Q4K_BLOCK_BYTES) / 4) * 4;
|
|
209
226
|
bRequired = bOffset + bBindingSize;
|
|
227
|
+
} else if (bDtype === 'litert_int4') {
|
|
228
|
+
const packedBytesPerRow = Math.ceil(K / 2);
|
|
229
|
+
bBindingSize = Math.ceil((N * packedBytesPerRow) / 4) * 4;
|
|
230
|
+
bRequired = bOffset + bBindingSize;
|
|
231
|
+
} else if (bDtype === 'w4a16') {
|
|
232
|
+
const groupsPerRow = Math.ceil(K / 32);
|
|
233
|
+
bBindingSize = Math.ceil((N * groupsPerRow * 16) / 4) * 4;
|
|
234
|
+
bRequired = bOffset + bBindingSize;
|
|
210
235
|
} else {
|
|
211
236
|
const bBytesPerElem = bDtype === 'f16' ? 2 : 4;
|
|
212
237
|
const bElements = transposeB ? N * K : K * N;
|
|
@@ -229,6 +254,14 @@ function isQ4KFusedVariant(variant) {
|
|
|
229
254
|
return variant.startsWith('q4_fused');
|
|
230
255
|
}
|
|
231
256
|
|
|
257
|
+
function isLiteRTInt4FusedVariant(variant) {
|
|
258
|
+
return variant.startsWith('litert_int4_');
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
function isW4A16FusedVariant(variant) {
|
|
262
|
+
return variant.startsWith('w4a16_');
|
|
263
|
+
}
|
|
264
|
+
|
|
232
265
|
|
|
233
266
|
function isGemvVariant(variant) {
|
|
234
267
|
return variant.startsWith('gemv');
|
|
@@ -236,7 +269,11 @@ function isGemvVariant(variant) {
|
|
|
236
269
|
|
|
237
270
|
|
|
238
271
|
function supportsF16Input(variant) {
|
|
239
|
-
return variant === 'f16'
|
|
272
|
+
return variant === 'f16'
|
|
273
|
+
|| variant === 'f16_vec4'
|
|
274
|
+
|| variant === 'f16_tiled'
|
|
275
|
+
|| variant.endsWith('_f16a')
|
|
276
|
+
|| variant.includes('_f16a_');
|
|
240
277
|
}
|
|
241
278
|
|
|
242
279
|
export function requiresF32Input(variant) {
|
|
@@ -254,6 +291,7 @@ function resolveMatmulOverride(
|
|
|
254
291
|
K,
|
|
255
292
|
aDtype,
|
|
256
293
|
bDtype,
|
|
294
|
+
transposeB,
|
|
257
295
|
requestedOutputDtype,
|
|
258
296
|
capabilities,
|
|
259
297
|
strict,
|
|
@@ -319,6 +357,7 @@ function resolveMatmulOverride(
|
|
|
319
357
|
}
|
|
320
358
|
|
|
321
359
|
const useQ4KFused = isQ4KFusedVariant(override);
|
|
360
|
+
const useLiteRTInt4Fused = isLiteRTInt4FusedVariant(override);
|
|
322
361
|
if (useQ4KFused) {
|
|
323
362
|
if (bDtype !== 'q4k') {
|
|
324
363
|
return failOrWarn(`Matmul kernel "${variantOverride}" requires Q4K weights but B dtype is ${bDtype}.`);
|
|
@@ -327,13 +366,30 @@ function resolveMatmulOverride(
|
|
|
327
366
|
return failOrWarn(`Matmul kernel "${variantOverride}" blocked by kernel path (fused Q4K disabled).`);
|
|
328
367
|
}
|
|
329
368
|
}
|
|
369
|
+
if (useLiteRTInt4Fused) {
|
|
370
|
+
if (bDtype !== 'litert_int4') {
|
|
371
|
+
return failOrWarn(`Matmul kernel "${variantOverride}" requires LiteRT INT4 weights but B dtype is ${bDtype}.`);
|
|
372
|
+
}
|
|
373
|
+
if (transposeB !== true) {
|
|
374
|
+
return failOrWarn(`Matmul kernel "${variantOverride}" requires transposeB=true for row-packed [N,K] weights.`);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
const useW4A16Fused = isW4A16FusedVariant(override);
|
|
378
|
+
if (useW4A16Fused) {
|
|
379
|
+
if (bDtype !== 'w4a16') {
|
|
380
|
+
return failOrWarn(`Matmul kernel "${variantOverride}" requires W4A16 weights but B dtype is ${bDtype}.`);
|
|
381
|
+
}
|
|
382
|
+
if (transposeB !== true) {
|
|
383
|
+
return failOrWarn(`Matmul kernel "${variantOverride}" requires transposeB=true for row-packed [N,K] weights.`);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
330
386
|
|
|
331
387
|
const useGemv = isGemvVariant(override);
|
|
332
388
|
if (useGemv && M !== 1) {
|
|
333
389
|
return failOrWarn(`Matmul kernel "${variantOverride}" requires M=1 but got M=${M}.`);
|
|
334
390
|
}
|
|
335
391
|
|
|
336
|
-
return { variant: override, useQ4KFused, useGemv };
|
|
392
|
+
return { variant: override, useQ4KFused, useGemv, useLiteRTInt4Fused, useW4A16Fused };
|
|
337
393
|
}
|
|
338
394
|
|
|
339
395
|
function resolveGemvPathVariant(pathVariant, aDtype, requestedOutputDtype, N, multicolThreshold) {
|
|
@@ -378,19 +434,20 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
|
|
|
378
434
|
K,
|
|
379
435
|
aDtype,
|
|
380
436
|
bDtype,
|
|
437
|
+
transposeB,
|
|
381
438
|
requestedOutputDtype,
|
|
382
439
|
capabilities,
|
|
383
440
|
strict,
|
|
384
441
|
fusedQ4KDisabled
|
|
385
442
|
);
|
|
386
443
|
if (!override && strict) {
|
|
387
|
-
// When weights resolved to
|
|
444
|
+
// When weights resolved to packed quantized storage but the path variant requires F16,
|
|
388
445
|
// fall through to auto-selection rather than throwing. The auto path will pick
|
|
389
|
-
// the correct fused
|
|
390
|
-
if (bDtype === 'q4k') {
|
|
446
|
+
// the correct fused variant for the actual weight dtype.
|
|
447
|
+
if (bDtype === 'q4k' || bDtype === 'litert_int4' || bDtype === 'w4a16') {
|
|
391
448
|
logKernelSelectionOnce('matmul', {
|
|
392
449
|
variant: pathVariant,
|
|
393
|
-
reason: '
|
|
450
|
+
reason: 'path_override_quantized_fallthrough',
|
|
394
451
|
});
|
|
395
452
|
} else {
|
|
396
453
|
throw new Error(`[Matmul] Path variant "${pathVariant}" rejected for role=${options.role ?? '?'} layerIdx=${options.layerIdx ?? '?'} phase=${phase} M=${M} K=${K} aDtype=${aDtype} bDtype=${bDtype} outDtype=${requestedOutputDtype}`);
|
|
@@ -421,6 +478,7 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
|
|
|
421
478
|
variant: adaptiveVariant,
|
|
422
479
|
useQ4KFused: false,
|
|
423
480
|
useGemv: false,
|
|
481
|
+
useLiteRTInt4Fused: false,
|
|
424
482
|
};
|
|
425
483
|
logKernelSelectionOnce('matmul', {
|
|
426
484
|
variant: adaptiveSelection.variant,
|
|
@@ -440,7 +498,29 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
|
|
|
440
498
|
|
|
441
499
|
const fusedAllowed = !fusedQ4KDisabled;
|
|
442
500
|
const isQ4K = bDtype === 'q4k';
|
|
501
|
+
const isLiteRTInt4 = bDtype === 'litert_int4';
|
|
502
|
+
const isW4A16 = bDtype === 'w4a16';
|
|
443
503
|
const wantF16Output = requestedOutputDtype === 'f16' && capabilities.hasF16;
|
|
504
|
+
const litertInt4Variant = isLiteRTInt4 && capabilities.hasF16
|
|
505
|
+
? selectLiteRTInt4FusedVariant(requestedOutputDtype, aDtype, transposeB)
|
|
506
|
+
: null;
|
|
507
|
+
const useLiteRTInt4Fused = litertInt4Variant != null;
|
|
508
|
+
const w4a16Variant = isW4A16 && capabilities.hasF16
|
|
509
|
+
? selectW4A16FusedVariant(requestedOutputDtype, aDtype, transposeB, M > 1)
|
|
510
|
+
: null;
|
|
511
|
+
const useW4A16Fused = w4a16Variant != null;
|
|
512
|
+
if (isLiteRTInt4 && !useLiteRTInt4Fused) {
|
|
513
|
+
throw new Error(
|
|
514
|
+
`[Matmul] LiteRT INT4 weights require a fused kernel path. ` +
|
|
515
|
+
`No variant matched M=${M} K=${K} aDtype=${aDtype} output=${requestedOutputDtype} transposeB=${transposeB}.`
|
|
516
|
+
);
|
|
517
|
+
}
|
|
518
|
+
if (isW4A16 && !useW4A16Fused) {
|
|
519
|
+
throw new Error(
|
|
520
|
+
`[Matmul] W4A16 weights require a fused kernel path. ` +
|
|
521
|
+
`No variant matched M=${M} K=${K} aDtype=${aDtype} output=${requestedOutputDtype} transposeB=${transposeB}.`
|
|
522
|
+
);
|
|
523
|
+
}
|
|
444
524
|
let q4kVariant = isQ4K && capabilities.hasSubgroups && fusedAllowed
|
|
445
525
|
? selectQ4KFusedVariant(M === 1, wantF16Output, aDtype, phase)
|
|
446
526
|
: null;
|
|
@@ -488,6 +568,17 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
|
|
|
488
568
|
q4kVariant = 'q4_fused_widetile';
|
|
489
569
|
}
|
|
490
570
|
}
|
|
571
|
+
if (
|
|
572
|
+
options.useWideTileQ4KDecode === true
|
|
573
|
+
&& phase === 'decode'
|
|
574
|
+
&& M === 1
|
|
575
|
+
&& aDtype === 'f32'
|
|
576
|
+
&& !wantF16Output
|
|
577
|
+
&& capabilities.hasF16 === true
|
|
578
|
+
&& (q4kVariant === 'q4_fused_multicol' || q4kVariant === 'q4_fused')
|
|
579
|
+
) {
|
|
580
|
+
q4kVariant = 'q4_fused_widetile';
|
|
581
|
+
}
|
|
491
582
|
// Residual-fused WideTile override: when the caller passes a residualTensor
|
|
492
583
|
// and opts into the fusion via useWideTileResidualFusion, route the
|
|
493
584
|
// f32-output WideTile variant to its residual-epilogue twin. Saves one
|
|
@@ -512,7 +603,7 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
|
|
|
512
603
|
q4kVariant = 'q4_fused_rmsnorm_widetile';
|
|
513
604
|
}
|
|
514
605
|
|
|
515
|
-
const effectiveBDtype = bDtype === 'q4k' ? 'f32' : bDtype;
|
|
606
|
+
const effectiveBDtype = (bDtype === 'q4k' || bDtype === 'litert_int4' || bDtype === 'w4a16') ? 'f32' : bDtype;
|
|
516
607
|
const matmulVariant = selectMatmulKernel({
|
|
517
608
|
...options,
|
|
518
609
|
aDtype: aDtype === 'q4k' ? 'f32' : aDtype,
|
|
@@ -537,9 +628,13 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
|
|
|
537
628
|
const selection = selectKernelRuleValue(
|
|
538
629
|
'matmul',
|
|
539
630
|
'matmulSelection',
|
|
540
|
-
{ isQ4K, hasSubgroups: capabilities.hasSubgroups, fusedAllowed, useGemv, q4kVariant, gemvVariant, matmulVariant }
|
|
631
|
+
{ isQ4K, isLiteRTInt4, isW4A16, hasSubgroups: capabilities.hasSubgroups, fusedAllowed, useGemv, useLiteRTInt4Fused, useW4A16Fused, litertInt4Variant, w4a16Variant, q4kVariant, gemvVariant, matmulVariant }
|
|
541
632
|
);
|
|
542
|
-
const reason = selection.
|
|
633
|
+
const reason = selection.useLiteRTInt4Fused
|
|
634
|
+
? 'litert_int4_fused'
|
|
635
|
+
: selection.useW4A16Fused
|
|
636
|
+
? 'w4a16_fused'
|
|
637
|
+
: selection.useQ4KFused
|
|
543
638
|
? 'q4k_fused'
|
|
544
639
|
: selection.useGemv
|
|
545
640
|
? 'gemv'
|
|
@@ -38,7 +38,7 @@ export interface MatmulOptions extends OutputBufferOptions, OutputDtypeOptions,
|
|
|
38
38
|
bOffset?: number;
|
|
39
39
|
cOffset?: number;
|
|
40
40
|
aDtype?: 'f16' | 'f32' | null;
|
|
41
|
-
bDtype?: 'f16' | 'f32' | 'q4k' | null;
|
|
41
|
+
bDtype?: 'f16' | 'f32' | 'q4k' | 'litert_int4' | 'w4a16' | null;
|
|
42
42
|
preferF16?: boolean;
|
|
43
43
|
/** WGSL override constants for pipeline creation */
|
|
44
44
|
constants?: Record<string, number | boolean>;
|
|
@@ -3,6 +3,7 @@ import { createTensor } from '../tensor.js';
|
|
|
3
3
|
import {
|
|
4
4
|
getBuffer,
|
|
5
5
|
getLayout,
|
|
6
|
+
getWeightMetadata,
|
|
6
7
|
getWeightDtype,
|
|
7
8
|
isWeightBuffer,
|
|
8
9
|
resolveWeightBufferMaterialization,
|
|
@@ -37,7 +38,7 @@ import {
|
|
|
37
38
|
createMatmulBindGroupLayout,
|
|
38
39
|
getMatmulPipeline,
|
|
39
40
|
} from './matmul-dispatch.js';
|
|
40
|
-
import { __dbgRecord } from './utils.js';
|
|
41
|
+
import { RECORD_STAGE_DEBUG_ENABLED, __dbgRecord, getPipelineBindGroupLayout } from './utils.js';
|
|
41
42
|
|
|
42
43
|
export { isFusedQ4KDisabled, selectMatmulKernel };
|
|
43
44
|
export { createMatmulBindGroupLayout };
|
|
@@ -81,6 +82,36 @@ function buildProfileLabel(options = {}) {
|
|
|
81
82
|
return `matmul${roleLabel}${layerLabel}`;
|
|
82
83
|
}
|
|
83
84
|
|
|
85
|
+
function resolveLiteRTInt4StorageEncoding(weight, label) {
|
|
86
|
+
const storageEncoding = String(getWeightMetadata(weight)?.storageEncoding ?? '').toLowerCase();
|
|
87
|
+
if (storageEncoding !== 'signed' && storageEncoding !== 'offset_binary') {
|
|
88
|
+
throw new Error(
|
|
89
|
+
`[Matmul] LiteRT INT4 weight "${label ?? 'unknown'}" requires metadata.storageEncoding ` +
|
|
90
|
+
`"signed" or "offset_binary", got "${storageEncoding || 'missing'}".`
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
return storageEncoding;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function resolveW4A16ScaleDtype(weight, label) {
|
|
97
|
+
const metadata = getWeightMetadata(weight);
|
|
98
|
+
const scaleDtype = String(metadata?.scaleDtype ?? '').toLowerCase();
|
|
99
|
+
if (scaleDtype !== 'f16' && scaleDtype !== 'bf16' && scaleDtype !== 'f32') {
|
|
100
|
+
throw new Error(
|
|
101
|
+
`[Matmul] W4A16 weight "${label ?? 'unknown'}" requires metadata.scaleDtype ` +
|
|
102
|
+
`"f16", "bf16", or "f32", got "${scaleDtype || 'missing'}".`
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
return scaleDtype;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function resolveW4A16ScaleDtypeConstant(scaleDtype) {
|
|
109
|
+
if (scaleDtype === 'f16') return 0;
|
|
110
|
+
if (scaleDtype === 'bf16') return 1;
|
|
111
|
+
if (scaleDtype === 'f32') return 2;
|
|
112
|
+
throw new Error(`[Matmul] unsupported W4A16 scale dtype "${scaleDtype}".`);
|
|
113
|
+
}
|
|
114
|
+
|
|
84
115
|
function assertBindGroupBuffer(kernelName, variant, bindingIndex, bindingLabel, buffer, details = []) {
|
|
85
116
|
const isGpuBuffer = buffer && (
|
|
86
117
|
typeof GPUBuffer === 'undefined'
|
|
@@ -98,7 +129,7 @@ function assertBindGroupBuffer(kernelName, variant, bindingIndex, bindingLabel,
|
|
|
98
129
|
);
|
|
99
130
|
}
|
|
100
131
|
|
|
101
|
-
function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuffer, outputBuffer, offsets, bindingSizes, residualBuffer = null, normWeightBuffer = null) {
|
|
132
|
+
function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuffer, outputBuffer, offsets, bindingSizes, residualBuffer = null, normWeightBuffer = null, scaleBuffer = null) {
|
|
102
133
|
const isQ4KF16 = variant === 'q4_fused_multicol_f16'
|
|
103
134
|
|| variant === 'q4_fused_f16a'
|
|
104
135
|
|| variant === 'q4_fused_batched_f16'
|
|
@@ -109,11 +140,12 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
|
|
|
109
140
|
|| variant === 'q4_fused_prefill_tiled_f16'
|
|
110
141
|
|| variant === 'q4_fused_widetile_f16'
|
|
111
142
|
|| variant === 'q4_fused_widetile_f16a';
|
|
112
|
-
// 5-entry
|
|
143
|
+
// 5-entry Q4K epilogue/prologue variants: output at binding 3 + one
|
|
113
144
|
// extra read-only buffer at binding 4 (residual for _residual, norm weight
|
|
114
145
|
// for _rmsnorm). Distinct from isQ4KF16 (which puts output at binding 4).
|
|
115
|
-
const
|
|
146
|
+
const isQ4KResidual = variant === 'q4_fused_widetile_residual';
|
|
116
147
|
const isWideTileRmsnorm = variant === 'q4_fused_rmsnorm_widetile';
|
|
148
|
+
const isW4A16 = variant.startsWith('w4a16_');
|
|
117
149
|
|
|
118
150
|
assertBindGroupBuffer('matmul', variant, 0, 'uniforms', uniformBuffer);
|
|
119
151
|
assertBindGroupBuffer('matmul', variant, 1, 'input', matmulInput?.buffer, [
|
|
@@ -121,8 +153,11 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
|
|
|
121
153
|
`inputDtype=${matmulInput?.dtype ?? 'unknown'}`,
|
|
122
154
|
]);
|
|
123
155
|
assertBindGroupBuffer('matmul', variant, 2, 'weights', bBuffer);
|
|
124
|
-
|
|
125
|
-
|
|
156
|
+
if (isW4A16) {
|
|
157
|
+
assertBindGroupBuffer('matmul', variant, 3, 'scales', scaleBuffer);
|
|
158
|
+
}
|
|
159
|
+
assertBindGroupBuffer('matmul', variant, (isQ4KF16 || isW4A16) ? 4 : 3, 'output', outputBuffer);
|
|
160
|
+
if (isQ4KResidual) {
|
|
126
161
|
if (!residualBuffer) {
|
|
127
162
|
throw new Error(`[Matmul] variant "${variant}" requires a residual buffer but none was provided.`);
|
|
128
163
|
}
|
|
@@ -141,7 +176,16 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
|
|
|
141
176
|
{ binding: 2, resource: { buffer: bBuffer, offset: offsets.bOffset, size: bindingSizes.bBindingSize } },
|
|
142
177
|
];
|
|
143
178
|
|
|
144
|
-
if (
|
|
179
|
+
if (isW4A16) {
|
|
180
|
+
entries.push({
|
|
181
|
+
binding: 3,
|
|
182
|
+
resource: { buffer: scaleBuffer },
|
|
183
|
+
});
|
|
184
|
+
entries.push({
|
|
185
|
+
binding: 4,
|
|
186
|
+
resource: { buffer: outputBuffer, offset: offsets.cOffset, size: bindingSizes.cBindingSize },
|
|
187
|
+
});
|
|
188
|
+
} else if (isQ4KF16) {
|
|
145
189
|
entries.push({
|
|
146
190
|
binding: 4,
|
|
147
191
|
resource: { buffer: outputBuffer, offset: offsets.cOffset, size: bindingSizes.cBindingSize },
|
|
@@ -151,7 +195,7 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
|
|
|
151
195
|
binding: 3,
|
|
152
196
|
resource: { buffer: outputBuffer, offset: offsets.cOffset, size: bindingSizes.cBindingSize },
|
|
153
197
|
});
|
|
154
|
-
if (
|
|
198
|
+
if (isQ4KResidual) {
|
|
155
199
|
entries.push({
|
|
156
200
|
binding: 4,
|
|
157
201
|
resource: { buffer: residualBuffer },
|
|
@@ -254,6 +298,7 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
254
298
|
const effectiveOptions = (
|
|
255
299
|
options.useTiledQ4KPrefill == null
|
|
256
300
|
|| options.useWideTileQ4KPrefill == null
|
|
301
|
+
|| options.useWideTileQ4KDecode == null
|
|
257
302
|
|| options.useWideTileResidualFusion == null
|
|
258
303
|
|| options.useFusedRmsnormWideTile == null
|
|
259
304
|
)
|
|
@@ -261,12 +306,13 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
261
306
|
...options,
|
|
262
307
|
useTiledQ4KPrefill: options.useTiledQ4KPrefill ?? (runtimeSession?.useTiledQ4KPrefill === true),
|
|
263
308
|
useWideTileQ4KPrefill: options.useWideTileQ4KPrefill ?? (runtimeSession?.useWideTileQ4KPrefill === true),
|
|
309
|
+
useWideTileQ4KDecode: options.useWideTileQ4KDecode ?? (runtimeSession?.useWideTileQ4KDecode === true),
|
|
264
310
|
useWideTileResidualFusion: options.useWideTileResidualFusion ?? (runtimeSession?.useWideTileResidualFusion === true),
|
|
265
311
|
useFusedRmsnormWideTile: options.useFusedRmsnormWideTile ?? (runtimeSession?.useFusedRmsnormWideTile === true),
|
|
266
312
|
}
|
|
267
313
|
: options;
|
|
268
314
|
|
|
269
|
-
let { variant, useQ4KFused, useGemv } = selectMatmulVariantAndFlags(
|
|
315
|
+
let { variant, useQ4KFused, useGemv, useLiteRTInt4Fused = false, useW4A16Fused = false } = selectMatmulVariantAndFlags(
|
|
270
316
|
mode,
|
|
271
317
|
M,
|
|
272
318
|
N,
|
|
@@ -290,6 +336,23 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
290
336
|
if (variant === 'f32' && constants && options.constants == null) {
|
|
291
337
|
constants = null;
|
|
292
338
|
}
|
|
339
|
+
if (bDtype === 'litert_int4') {
|
|
340
|
+
const storageEncoding = resolveLiteRTInt4StorageEncoding(resolvedWeight, weightLabel);
|
|
341
|
+
constants = {
|
|
342
|
+
...(constants ?? {}),
|
|
343
|
+
STORAGE_OFFSET_BINARY: storageEncoding === 'offset_binary' ? 1 : 0,
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
let w4a16ScaleBuffer = null;
|
|
347
|
+
if (bDtype === 'w4a16') {
|
|
348
|
+
const metadata = getWeightMetadata(resolvedWeight);
|
|
349
|
+
w4a16ScaleBuffer = metadata?.scaleBuffer ?? null;
|
|
350
|
+
const scaleDtype = resolveW4A16ScaleDtype(resolvedWeight, weightLabel);
|
|
351
|
+
constants = {
|
|
352
|
+
...(constants ?? {}),
|
|
353
|
+
SCALE_DTYPE: resolveW4A16ScaleDtypeConstant(scaleDtype),
|
|
354
|
+
};
|
|
355
|
+
}
|
|
293
356
|
// For the rmsnorm-fused WideTile variant, forward the caller's
|
|
294
357
|
// rmsNormOffset flag as a pipeline override constant. Gemma-family norm
|
|
295
358
|
// weights encode `(w - 1.0)`; other models encode `w`.
|
|
@@ -297,7 +360,10 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
297
360
|
// kernel correctly unpacks f16-packed weights (Gemma hidden weights) vs
|
|
298
361
|
// f32 weights.
|
|
299
362
|
if (variant === 'q4_fused_rmsnorm_widetile' && options.rmsNormOffset != null) {
|
|
300
|
-
const normWeightDtype = getWeightDtype(options.normWeight)
|
|
363
|
+
const normWeightDtype = getWeightDtype(options.normWeight);
|
|
364
|
+
if (!normWeightDtype) {
|
|
365
|
+
throw new Error('[Matmul] q4_fused_rmsnorm_widetile requires normWeight dtype metadata.');
|
|
366
|
+
}
|
|
301
367
|
constants = {
|
|
302
368
|
...(constants ?? {}),
|
|
303
369
|
RMS_NORM_OFFSET: options.rmsNormOffset === true,
|
|
@@ -372,12 +438,24 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
372
438
|
log.warn('MatmulQKVProbe', `${err.message} | ${probeDetail}`);
|
|
373
439
|
}
|
|
374
440
|
if (shouldFailOnSmallAttentionWeightBuffer) {
|
|
441
|
+
if (!isRecord && castedInput) {
|
|
442
|
+
releaseBuffer(castedInput.buffer);
|
|
443
|
+
castedInput = null;
|
|
444
|
+
}
|
|
375
445
|
throw new Error(`${err.message}${detail}`);
|
|
376
446
|
}
|
|
377
447
|
}
|
|
378
448
|
if (err instanceof Error && err.message.includes('B buffer too small')) {
|
|
449
|
+
if (!isRecord && castedInput) {
|
|
450
|
+
releaseBuffer(castedInput.buffer);
|
|
451
|
+
castedInput = null;
|
|
452
|
+
}
|
|
379
453
|
throw new Error(`${err.message}${detail}`);
|
|
380
454
|
}
|
|
455
|
+
if (!isRecord && castedInput) {
|
|
456
|
+
releaseBuffer(castedInput.buffer);
|
|
457
|
+
castedInput = null;
|
|
458
|
+
}
|
|
381
459
|
throw err;
|
|
382
460
|
}
|
|
383
461
|
|
|
@@ -402,39 +480,66 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
402
480
|
);
|
|
403
481
|
}
|
|
404
482
|
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
483
|
+
let __dbg = false;
|
|
484
|
+
let __t0 = 0;
|
|
485
|
+
let __tPipeline = 0;
|
|
486
|
+
let config;
|
|
487
|
+
let kernel;
|
|
488
|
+
let pipeline;
|
|
489
|
+
let C = null;
|
|
490
|
+
let outputSize;
|
|
491
|
+
let cBindingSize;
|
|
492
|
+
let actualOutputDtype;
|
|
493
|
+
let ownsOutput = false;
|
|
494
|
+
let dispatchPlan;
|
|
495
|
+
try {
|
|
496
|
+
__dbg = RECORD_STAGE_DEBUG_ENABLED;
|
|
497
|
+
__t0 = __dbg ? performance.now() : 0;
|
|
498
|
+
config = getMatmulConfig(variant, constants);
|
|
499
|
+
kernel = new MatmulKernel(device);
|
|
500
|
+
pipeline = await getMatmulPipeline(variant, constants);
|
|
501
|
+
__tPipeline = __dbg ? performance.now() : 0;
|
|
502
|
+
|
|
503
|
+
const outputInfo = resolveMatmulOutput(
|
|
504
|
+
variant,
|
|
505
|
+
M,
|
|
506
|
+
N,
|
|
507
|
+
outputBuffer
|
|
425
508
|
);
|
|
426
|
-
|
|
509
|
+
C = outputInfo.output;
|
|
510
|
+
outputSize = outputInfo.outputSize;
|
|
511
|
+
cBindingSize = outputInfo.cBindingSize;
|
|
512
|
+
actualOutputDtype = outputInfo.actualOutputDtype;
|
|
513
|
+
ownsOutput = outputBuffer == null;
|
|
514
|
+
|
|
515
|
+
if (isAttnProj && shouldLogAttentionWeightBuffer) {
|
|
516
|
+
log.warn('MatmulVariantDiag',
|
|
517
|
+
`role=${options.role ?? ''} layer=${Number.isFinite(options.layerIdx) ? options.layerIdx : '?'} mode=${mode} ` +
|
|
518
|
+
`variant=${variant} useQ4KFused=${useQ4KFused} useGemv=${useGemv} useLiteRTInt4Fused=${useLiteRTInt4Fused} ` +
|
|
519
|
+
`aDtype=${aDtype} bDtype=${bDtype} output=${actualOutputDtype}`
|
|
520
|
+
);
|
|
521
|
+
}
|
|
427
522
|
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
523
|
+
if (!Number.isFinite(outputSize) || outputSize <= 0) {
|
|
524
|
+
throw new Error(`[${opLabel}] Invalid output size: ${outputSize} (M=${M}, N=${N})`);
|
|
525
|
+
}
|
|
431
526
|
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
527
|
+
const cRequired = cOffset + cBindingSize;
|
|
528
|
+
if (C.size < cRequired) {
|
|
529
|
+
throw new Error(`[${opLabel}] Output buffer too small: ${C.size} < ${cRequired} (M=${M}, N=${N})`);
|
|
530
|
+
}
|
|
436
531
|
|
|
437
|
-
|
|
532
|
+
dispatchPlan = calculateMatmulDispatch(variant, useQ4KFused, useGemv, useLiteRTInt4Fused, M, N, config, useW4A16Fused);
|
|
533
|
+
} catch (error) {
|
|
534
|
+
if (!isRecord && castedInput) {
|
|
535
|
+
releaseBuffer(castedInput.buffer);
|
|
536
|
+
castedInput = null;
|
|
537
|
+
}
|
|
538
|
+
if (ownsOutput && C) {
|
|
539
|
+
releaseBuffer(C);
|
|
540
|
+
}
|
|
541
|
+
throw error;
|
|
542
|
+
}
|
|
438
543
|
let uniformBuffer = null;
|
|
439
544
|
let completed = false;
|
|
440
545
|
try {
|
|
@@ -470,13 +575,14 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
470
575
|
cBindingSize,
|
|
471
576
|
},
|
|
472
577
|
residualBuffer,
|
|
473
|
-
normWeightBuffer
|
|
578
|
+
normWeightBuffer,
|
|
579
|
+
w4a16ScaleBuffer
|
|
474
580
|
);
|
|
475
581
|
|
|
476
582
|
const __tBgStart = __dbg ? performance.now() : 0;
|
|
477
583
|
const bindGroup = device.createBindGroup({
|
|
478
584
|
label: 'matmul_bind_group',
|
|
479
|
-
layout: pipeline
|
|
585
|
+
layout: getPipelineBindGroupLayout(pipeline, 0),
|
|
480
586
|
entries,
|
|
481
587
|
});
|
|
482
588
|
const __tBg = __dbg ? performance.now() : 0;
|
|
@@ -490,16 +596,18 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
490
596
|
const __tEnd = performance.now();
|
|
491
597
|
__dbgRecord('matmul', variant, __tPipeline - __t0, __tBgStart - __tPipeline, __tBg - __tBgStart, __tEnd - __tBg);
|
|
492
598
|
}
|
|
599
|
+
const tensor = createTensor(C, actualOutputDtype, [M, N], 'matmul_output');
|
|
493
600
|
completed = true;
|
|
494
|
-
return
|
|
601
|
+
return tensor;
|
|
495
602
|
} finally {
|
|
496
603
|
if (!isRecord && uniformBuffer) {
|
|
497
604
|
releaseUniformBuffer(uniformBuffer);
|
|
498
605
|
}
|
|
499
606
|
if (!isRecord && castedInput) {
|
|
500
607
|
releaseBuffer(castedInput.buffer);
|
|
608
|
+
castedInput = null;
|
|
501
609
|
}
|
|
502
|
-
if (!completed && ownsOutput) {
|
|
610
|
+
if (!completed && ownsOutput && C) {
|
|
503
611
|
releaseBuffer(C);
|
|
504
612
|
}
|
|
505
613
|
}
|