doppler-gpu 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -577
- package/LICENSE +21 -201
- package/README.md +165 -222
- package/models/catalog.json +241 -33
- package/package.json +99 -86
- package/src/capsule-runtime.d.ts +23 -0
- package/src/capsule-runtime.js +23 -0
- package/src/capsule.d.ts +8 -0
- package/src/capsule.js +4 -0
- package/src/cli/argument-contract.js +200 -0
- package/src/cli/browser-report-output.js +24 -0
- package/src/cli/cli-model-resolution.js +2 -2
- package/src/cli/cli-output.js +10 -0
- package/src/cli/commands/bundle.js +828 -0
- package/src/cli/doppler-cli.js +50 -1457
- package/src/cli/doppler-serve.js +18 -244
- package/src/cli/output/manifest.js +368 -0
- package/src/cli/runtime-profiles.js +1 -1
- package/src/cli/serve/capsule-handler.d.ts +33 -0
- package/src/cli/serve/capsule-handler.js +105 -0
- package/src/cli/serve/capsule-http.js +102 -0
- package/src/cli/serve/receipt.js +269 -0
- package/src/client/capsule-host.browser.js +1 -0
- package/src/client/capsule-host.d.ts +3 -0
- package/src/client/capsule-host.js +1 -0
- package/src/client/doppler-api.browser.js +42 -0
- package/src/client/doppler-api.d.ts +36 -1
- package/src/client/doppler-api.js +40 -1
- package/src/client/doppler-registry.js +15 -7
- package/src/client/electron/index.d.ts +4 -0
- package/src/client/electron/index.js +13 -0
- package/src/client/electron/ipc-contract.d.ts +9 -0
- package/src/client/electron/ipc-contract.js +56 -0
- package/src/client/electron/release-state.d.ts +53 -0
- package/src/client/electron/release-state.js +425 -0
- package/src/client/electron/renderer-runtime.d.ts +19 -0
- package/src/client/electron/renderer-runtime.js +104 -0
- package/src/client/failure-taxonomy.js +1 -28
- package/src/client/fault-injection.js +0 -8
- package/src/client/inspection.js +1 -1
- package/src/client/model-host/index.d.ts +140 -0
- package/src/client/model-host/index.js +548 -0
- package/src/client/model-host/model-evidence.js +255 -0
- package/src/client/model-host/model-session.d.ts +322 -0
- package/src/client/{runtime → model-host}/model-session.js +190 -148
- package/src/client/provider.d.ts +18 -2
- package/src/client/provider.js +26 -3
- package/src/client/receipt.js +30 -72
- package/src/client/runtime/capsule-acquisition.d.ts +18 -0
- package/src/client/runtime/capsule-acquisition.js +103 -0
- package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
- package/src/client/runtime/capsule-adapter-execution.js +63 -0
- package/src/client/runtime/capsule-artifact-source.js +62 -0
- package/src/client/runtime/capsule-embedding.d.ts +29 -0
- package/src/client/runtime/capsule-embedding.js +76 -0
- package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
- package/src/client/runtime/capsule-forecast-program.js +85 -0
- package/src/client/runtime/capsule-forecast.d.ts +17 -0
- package/src/client/runtime/capsule-forecast.js +31 -0
- package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
- package/src/client/runtime/capsule-operation-adapters.js +68 -0
- package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
- package/src/client/runtime/capsule-operation-executor.js +100 -0
- package/src/client/runtime/capsule-program-adapter.js +151 -0
- package/src/client/runtime/capsule-release-authorization.js +25 -0
- package/src/client/runtime/capsule-rerank.d.ts +50 -0
- package/src/client/runtime/capsule-rerank.js +105 -0
- package/src/client/runtime/capsule-session-execution.js +85 -0
- package/src/client/runtime/command-executor.d.ts +15 -0
- package/src/client/runtime/command-executor.js +128 -0
- package/src/client/runtime/composition-root.d.ts +74 -0
- package/src/client/runtime/composition-root.js +344 -0
- package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
- package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
- package/src/client/runtime/index.d.ts +1 -117
- package/src/client/runtime/index.js +1 -433
- package/src/client/runtime/lora.d.ts +14 -0
- package/src/client/runtime/lora.js +19 -1
- package/src/client/runtime/model-manager.js +12 -0
- package/src/client/runtime/model-rerank.js +30 -0
- package/src/client/runtime/model-session.d.ts +1 -221
- package/src/client/runtime/model-source.d.ts +9 -0
- package/src/client/runtime/model-source.js +46 -17
- package/src/client/runtime/node-quickstart-cache.js +1 -1
- package/src/client/runtime/resolution-policy.d.ts +20 -0
- package/src/client/runtime/resolution-policy.js +84 -0
- package/src/client/runtime/resource-binder.d.ts +24 -0
- package/src/client/runtime/resource-binder.js +135 -0
- package/src/client/runtime/scoped-session.d.ts +10 -1
- package/src/client/runtime/scoped-session.js +20 -1
- package/src/client/runtime/session-controller.d.ts +30 -0
- package/src/client/runtime/session-controller.js +170 -0
- package/src/client/runtime/source-runtime.js +1 -1
- package/src/client/runtime/target-selector.d.ts +9 -0
- package/src/client/runtime/target-selector.js +2 -0
- package/src/client/runtime/types.d.ts +1 -0
- package/src/client/runtime/verified-capsule-artifact-store.js +72 -0
- package/src/client/wrap-pipeline-handle.js +3 -9
- package/src/config/backward-registry-loader.js +1 -1
- package/src/config/capsule-adapter-policy.js +30 -0
- package/src/config/capsule-adapters.d.ts +21 -0
- package/src/config/capsule-adapters.js +40 -0
- package/src/config/capsule-adapters.json +9 -0
- package/src/config/capsule-loading.d.ts +2 -0
- package/src/config/capsule-loading.js +14 -0
- package/src/config/capsule-loading.schema.json +11 -0
- package/src/config/capsule-operation.d.ts +19 -0
- package/src/config/capsule-operation.js +58 -0
- package/src/config/capsule-operations.json +18 -0
- package/src/config/capsule-release-contract.d.ts +66 -0
- package/src/config/capsule-release-contract.js +197 -0
- package/src/config/capsule-release-events.d.ts +53 -0
- package/src/config/capsule-release-events.js +172 -0
- package/src/config/capsule-sequence-receipt.js +19 -0
- package/src/config/capsule-serve.d.ts +9 -0
- package/src/config/capsule-serve.js +28 -0
- package/src/config/capsule-serve.schema.json +14 -0
- package/src/config/capsule-signature.d.ts +6 -0
- package/src/config/capsule-signature.js +50 -0
- package/src/config/capsule-v2.d.ts +70 -0
- package/src/config/capsule-v2.js +431 -0
- package/src/config/capsule-v3.d.ts +19 -0
- package/src/config/capsule-v3.js +72 -0
- package/src/config/capsule.d.ts +22 -0
- package/src/config/capsule.js +52 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +41 -1
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +1 -1
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +59 -7
- package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
- package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
- package/src/config/dtype-transition-contract.js +40 -0
- package/src/config/embedding-contract.js +26 -0
- package/src/config/embedding-reference.js +85 -0
- package/src/config/evidence/source-intake-policy.json +1 -1
- package/src/config/execution-contract-check.d.ts +16 -73
- package/src/config/execution-contract-check.js +7 -429
- package/src/config/forecast-manifest.js +56 -0
- package/src/config/heterogeneous-attention-contract.js +46 -0
- package/src/config/initial-execution-identity.d.ts +58 -0
- package/src/config/initial-execution-identity.js +331 -0
- package/src/config/kernel-path-loader.d.ts +10 -0
- package/src/config/kernel-path-loader.js +23 -8
- package/src/config/kernel-registry-contract.d.ts +25 -0
- package/src/config/kernel-registry-contract.js +46 -0
- package/src/config/kernels/kernel-ref-digests.js +50 -2
- package/src/config/kernels/registry.js +1 -1
- package/src/config/kernels/registry.json +13592 -11377
- package/src/config/lora-layouts.d.ts +10 -0
- package/src/config/lora-layouts.js +17 -0
- package/src/config/lora-layouts.json +9 -0
- package/src/config/manifest-inference-validation.js +575 -0
- package/src/config/merge/execution-patches.js +12 -0
- package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
- package/src/config/merge/value-selection.js +38 -0
- package/src/config/merge.js +43 -11
- package/src/config/model-ir-v2.d.ts +57 -0
- package/src/config/model-ir-v2.js +258 -0
- package/src/config/model-ir.d.ts +49 -0
- package/src/config/model-ir.js +180 -0
- package/src/config/param-validator.js +1 -1
- package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
- package/src/config/platforms/loader.js +2 -1
- package/src/config/production-release-evidence.d.ts +23 -0
- package/src/config/production-release-evidence.js +423 -0
- package/src/config/production-release.js +321 -0
- package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
- package/src/config/required-inference-fields-contract-check.js +2 -2
- package/src/config/rerank-reference.d.ts +28 -0
- package/src/config/rerank-reference.js +89 -0
- package/src/config/revocation-policy.d.ts +65 -0
- package/src/config/revocation-policy.js +290 -0
- package/src/config/revocation-registry.json +11 -0
- package/src/config/revocation-updates.d.ts +68 -0
- package/src/config/revocation-updates.js +440 -0
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
- package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
- package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
- package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
- package/src/config/runtime-merge.js +1 -1
- package/src/config/runtime.js +1 -1
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/conversion-report.schema.d.ts +3 -0
- package/src/config/schema/conversion-report.schema.js +29 -0
- package/src/config/schema/converter.schema.d.ts +1 -0
- package/src/config/schema/debug.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +5 -5
- package/src/config/schema/emulation.schema.js +1 -1
- package/src/config/schema/execution-v1.schema.d.ts +3 -0
- package/src/config/schema/execution-v1.schema.js +19 -0
- package/src/config/schema/harness.schema.d.ts +13 -0
- package/src/config/schema/harness.schema.js +6 -0
- package/src/config/schema/inference.schema.d.ts +21 -2
- package/src/config/schema/kernel-thresholds.schema.js +1 -1
- package/src/config/schema/manifest.schema.d.ts +26 -7
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/program-bundle.schema.d.ts +52 -6
- package/src/config/schema/program-bundle.schema.js +145 -10
- package/src/config/sequence-reference.d.ts +26 -0
- package/src/config/sequence-reference.js +100 -0
- package/src/config/target-plan.d.ts +108 -0
- package/src/config/target-plan.js +411 -0
- package/src/config/training-defaults.js +1 -1
- package/src/config/transforms/capability-transform-resolver.js +4 -105
- package/src/config/transforms/execution-graph/dtype.js +591 -0
- package/src/config/transforms/execution-graph/fusion.js +436 -0
- package/src/config/transforms/execution-graph/validation.js +414 -0
- package/src/config/transforms/execution-graph-transforms.js +13 -1419
- package/src/converter/artifact-identity.js +475 -0
- package/src/converter/conversion-plan.js +1 -1
- package/src/converter/core.js +94 -2096
- package/src/converter/forge-candidate-evaluation.js +271 -0
- package/src/converter/forge-qualification.js +162 -0
- package/src/converter/forge-stages.js +798 -0
- package/src/converter/manifest-builder.js +683 -0
- package/src/converter/quantization-info.js +4 -1
- package/src/converter/quantizer.js +58 -9
- package/src/converter/shard-packer.d.ts +1 -1
- package/src/converter/shard-packer.js +2 -2
- package/src/converter/tensor-transform.js +640 -0
- package/src/converter/tokenizer-contract.js +243 -0
- package/src/converter/tokenizer-manifest.js +243 -0
- package/src/debug/history.js +0 -8
- package/src/debug/index.js +0 -10
- package/src/debug/tensor.js +1 -1
- package/src/debug/trace.js +0 -7
- package/src/experimental/adapters/adapter-manager.js +2 -0
- package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
- package/src/experimental/adapters/adapter-manifest.js +11 -0
- package/src/experimental/adapters/artifact-contract.d.ts +2 -2
- package/src/experimental/adapters/artifact-contract.js +5 -2
- package/src/experimental/adapters/lora-loader.d.ts +10 -1
- package/src/experimental/adapters/lora-loader.js +110 -48
- package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
- package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
- package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
- package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
- package/src/experimental/browser/browser-converter.js +11 -11
- package/src/experimental/diffusion/index.d.ts +1 -1
- package/src/experimental/diffusion/index.js +1 -1
- package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
- package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
- package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
- package/src/experimental/diffusion/vae.js +596 -0
- package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
- package/src/experimental/distribution/shard-delivery/retry.js +392 -0
- package/src/experimental/distribution/shard-delivery/transport.js +831 -0
- package/src/experimental/distribution/shard-delivery.js +4 -1709
- package/src/experimental/energy/index.d.ts +2 -2
- package/src/experimental/energy/index.js +2 -2
- package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
- package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
- package/src/experimental/logits/cpu-output.d.ts +27 -0
- package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
- package/src/experimental/logits/cpu.d.ts +87 -0
- package/src/experimental/logits/cpu.js +95 -0
- package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
- package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
- package/src/experimental/orchestration/index.d.ts +1 -1
- package/src/experimental/orchestration/index.js +1 -1
- package/src/experimental/training/artifacts.js +2 -2
- package/src/experimental/training/checkpoint.js +2 -2
- package/src/experimental/training/distillation/artifacts.js +1 -1
- package/src/experimental/training/distillation/dataset.js +2 -2
- package/src/experimental/training/distillation/eval.js +1 -1
- package/src/experimental/training/distillation/runtime.js +2 -2
- package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
- package/src/experimental/training/distillation/student-fixture/model.js +353 -0
- package/src/experimental/training/distillation/student-fixture.js +2 -469
- package/src/experimental/training/lora/checkpoint.js +746 -0
- package/src/experimental/training/lora/execution.js +550 -0
- package/src/experimental/training/lora/plan.js +387 -0
- package/src/experimental/training/lora/recovery.js +418 -0
- package/src/experimental/training/lora-pipeline.js +7 -1949
- package/src/experimental/training/operator-artifacts.js +2 -2
- package/src/experimental/training/operator-command.js +4 -4
- package/src/experimental/training/runner/evaluation.js +62 -0
- package/src/experimental/training/runner/lifecycle.js +257 -0
- package/src/experimental/training/runner.js +5 -268
- package/src/experimental/training/suite/execution.js +879 -0
- package/src/experimental/training/suite/plan.js +855 -0
- package/src/experimental/training/suite.js +5 -1659
- package/src/experimental/training/workloads.d.ts +5 -5
- package/src/experimental/training/workloads.js +6 -6
- package/src/experimental/turboquant/codebook.js +470 -0
- package/src/formats/quantization-constants.d.ts +11 -0
- package/src/formats/quantization-constants.js +11 -0
- package/src/formats/rdrr/integrity.js +1 -1
- package/src/formats/rdrr/lowerings.js +0 -29
- package/src/formats/rdrr/merkle.js +1 -1
- package/src/formats/rdrr/tensor-config-validator.js +25 -5
- package/src/formats/rdrr/validation.js +3 -3
- package/src/gpu/command-recorder.d.ts +10 -1
- package/src/gpu/command-recorder.js +58 -14
- package/src/gpu/constant-buffer.js +29 -0
- package/src/gpu/device-state.js +36 -0
- package/src/gpu/device.js +6 -41
- package/src/gpu/kernel-runtime.js +2 -1
- package/src/gpu/kernels/activation-static-qdq.js +1 -1
- package/src/gpu/kernels/append_embedding.wgsl +14 -0
- package/src/gpu/kernels/attention/executors.js +960 -0
- package/src/gpu/kernels/attention/plan.js +472 -0
- package/src/gpu/kernels/attention.d.ts +4 -0
- package/src/gpu/kernels/attention.js +145 -1545
- package/src/gpu/kernels/attention_relative.wgsl +59 -0
- package/src/gpu/kernels/backward/adam.js +2 -1
- package/src/gpu/kernels/backward/attention_backward.js +1 -1
- package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
- package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
- package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
- package/src/gpu/kernels/backward/embed_backward.js +2 -1
- package/src/gpu/kernels/backward/executor.d.ts +34 -0
- package/src/gpu/kernels/backward/executor.js +158 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
- package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/gelu_backward.js +1 -1
- package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
- package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
- package/src/gpu/kernels/backward/matmul.js +133 -0
- package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
- package/src/gpu/kernels/backward/matmul_backward.js +1 -1
- package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
- package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
- package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/rope_backward.js +2 -1
- package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/scale_backward.js +2 -1
- package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/silu_backward.js +1 -1
- package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/softmax_backward.js +1 -1
- package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
- package/src/gpu/kernels/backward/utils.js +2 -392
- package/src/gpu/kernels/bind-group-validation.js +16 -0
- package/src/gpu/kernels/cast.js +2 -1
- package/src/gpu/kernels/check-finiteness.js +13 -49
- package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
- package/src/gpu/kernels/check-stop.js +14 -33
- package/src/gpu/kernels/check_finiteness.wgsl +33 -0
- package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
- package/src/gpu/kernels/check_stop.wgsl +20 -0
- package/src/gpu/kernels/clamp.js +1 -1
- package/src/gpu/kernels/conv2d.js +1 -1
- package/src/gpu/kernels/cross_entropy_loss.js +1 -1
- package/src/gpu/kernels/depthwise_conv1d.js +1 -1
- package/src/gpu/kernels/depthwise_conv2d.js +1 -1
- package/src/gpu/kernels/dequant.js +7 -2
- package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
- package/src/gpu/kernels/dispatch.js +12 -1
- package/src/gpu/kernels/embedding-pool.js +72 -0
- package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
- package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
- package/src/gpu/kernels/energy.js +2 -1
- package/src/gpu/kernels/forecast_output.wgsl +17 -0
- package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
- package/src/gpu/kernels/fused_ffn.js +10 -220
- package/src/gpu/kernels/fused_matmul_residual.js +2 -1
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
- package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
- package/src/gpu/kernels/gated-short-conv.js +11 -69
- package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
- package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
- package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
- package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
- package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
- package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
- package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
- package/src/gpu/kernels/gather.d.ts +1 -1
- package/src/gpu/kernels/gather.js +30 -6
- package/src/gpu/kernels/gather_bf16.wgsl +69 -0
- package/src/gpu/kernels/gelu.js +1 -1
- package/src/gpu/kernels/gemma4-route-expert.js +1 -1
- package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
- package/src/gpu/kernels/groupnorm.js +1 -1
- package/src/gpu/kernels/index.d.ts +38 -5
- package/src/gpu/kernels/index.js +18 -4
- package/src/gpu/kernels/instance_stats.wgsl +26 -0
- package/src/gpu/kernels/kernel-base.js +1 -1
- package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
- package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
- package/src/gpu/kernels/kernel-configs.d.ts +10 -61
- package/src/gpu/kernels/kernel-configs.js +26 -54
- package/src/gpu/kernels/kernel-execution.js +141 -0
- package/src/gpu/kernels/kv-cache-write.js +4 -4
- package/src/gpu/kernels/kv-quantize.js +4 -1
- package/src/gpu/kernels/l2-normalize.js +36 -0
- package/src/gpu/kernels/l2_normalize.wgsl +53 -0
- package/src/gpu/kernels/layernorm.d.ts +4 -4
- package/src/gpu/kernels/layernorm.js +44 -7
- package/src/gpu/kernels/linear-attention-core.js +29 -449
- package/src/gpu/kernels/linear_attention.js +1 -1
- package/src/gpu/kernels/lm-head-argmax.js +2 -2
- package/src/gpu/kernels/lm-head-select-logits.js +2 -2
- package/src/gpu/kernels/logit-finalize.js +103 -0
- package/src/gpu/kernels/logit-merge.js +13 -125
- package/src/gpu/kernels/logit-scatter.js +40 -0
- package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
- package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
- package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
- package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
- package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
- package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
- package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
- package/src/gpu/kernels/matmul-dispatch.js +6 -1
- package/src/gpu/kernels/matmul-selection.js +3 -2
- package/src/gpu/kernels/matmul.js +4 -15
- package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
- package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
- package/src/gpu/kernels/modulate.js +1 -1
- package/src/gpu/kernels/moe.js +3 -1
- package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
- package/src/gpu/kernels/patch_context.wgsl +29 -0
- package/src/gpu/kernels/pipeline-cache.js +3 -1
- package/src/gpu/kernels/pixel_shuffle.js +1 -1
- package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
- package/src/gpu/kernels/relu.js +1 -1
- package/src/gpu/kernels/repeat_channels.js +1 -1
- package/src/gpu/kernels/residual.js +2 -2
- package/src/gpu/kernels/rmsnorm-pair.js +1 -1
- package/src/gpu/kernels/rmsnorm-qk.js +1 -1
- package/src/gpu/kernels/rmsnorm-stats.js +14 -113
- package/src/gpu/kernels/rmsnorm.js +1 -1
- package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
- package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
- package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
- package/src/gpu/kernels/rope-precompute.js +160 -0
- package/src/gpu/kernels/rope-qk.js +1 -1
- package/src/gpu/kernels/rope.js +2 -2
- package/src/gpu/kernels/rope_precompute.wgsl +74 -0
- package/src/gpu/kernels/sample.js +5 -1
- package/src/gpu/kernels/scale.js +1 -1
- package/src/gpu/kernels/shader-cache.js +8 -4
- package/src/gpu/kernels/shader-source-scope.js +68 -0
- package/src/gpu/kernels/silu.js +2 -1
- package/src/gpu/kernels/soft-embedding.js +3 -1
- package/src/gpu/kernels/softmax.js +4 -2
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
- package/src/gpu/kernels/split_qg.js +1 -1
- package/src/gpu/kernels/split_qkv.d.ts +2 -0
- package/src/gpu/kernels/split_qkv.js +18 -3
- package/src/gpu/kernels/split_qkv.wgsl +5 -2
- package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
- package/src/gpu/kernels/submit_probe.wgsl +6 -0
- package/src/gpu/kernels/transpose.js +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +4 -535
- package/src/gpu/kernels/uniform-utils.js +7 -2
- package/src/gpu/kernels/upsample2d.js +1 -1
- package/src/gpu/kernels/utils.js +9 -330
- package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
- package/src/gpu/kernels/vision-average-pool.js +68 -0
- package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
- package/src/gpu/kernels/vision-patch-embed.js +117 -0
- package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
- package/src/gpu/kernels/vision-position-embedding.js +79 -0
- package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
- package/src/gpu/kernels/vision-rope-2d.js +73 -0
- package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
- package/src/gpu/kernels/vision-spatial-merge.js +100 -0
- package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
- package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
- package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
- package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
- package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
- package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
- package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
- package/src/gpu/readback-buffer.js +20 -0
- package/src/gpu/submit-probe.js +2 -26
- package/src/gpu/weight-buffer.d.ts +1 -0
- package/src/gpu/weight-buffer.js +7 -0
- package/src/index.d.ts +10 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness/benchmark-execution.js +872 -0
- package/src/inference/browser-harness/benchmark-rerank.js +284 -0
- package/src/inference/browser-harness/decode-diagnostics.js +241 -0
- package/src/inference/browser-harness/diffusion-performance.js +102 -0
- package/src/inference/browser-harness/inference-execution.js +543 -0
- package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
- package/src/inference/browser-harness/model-resolution.js +137 -0
- package/src/inference/browser-harness/report.js +338 -0
- package/src/inference/browser-harness/request.js +263 -0
- package/src/inference/browser-harness/runtime-config.d.ts +25 -0
- package/src/inference/browser-harness/runtime-config.js +255 -0
- package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
- package/src/inference/browser-harness/runtime-isolation.js +45 -0
- package/src/inference/browser-harness/runtime-report.d.ts +26 -0
- package/src/inference/browser-harness/runtime-report.js +117 -0
- package/src/inference/browser-harness/suite-summary.js +39 -0
- package/src/inference/browser-harness/text-evidence.js +694 -0
- package/src/inference/browser-harness/text-execution.js +757 -0
- package/src/inference/browser-harness/text-input.js +725 -0
- package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
- package/src/inference/browser-harness-contract-helpers.js +1 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
- package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
- package/src/inference/browser-harness-runtime-helpers.js +3 -423
- package/src/inference/browser-harness-text-helpers.js +31 -2199
- package/src/inference/browser-harness.d.ts +15 -2
- package/src/inference/browser-harness.js +38 -1952
- package/src/inference/kv-cache/base.js +67 -381
- package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
- package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
- package/src/inference/kv-cache/quantized.d.ts +2 -2
- package/src/inference/kv-cache/quantized.js +4 -8
- package/src/inference/kv-cache/update.js +309 -0
- package/src/inference/model-checkpoint-evidence.d.ts +75 -0
- package/src/inference/model-checkpoint-evidence.js +264 -0
- package/src/inference/moe-router.js +36 -216
- package/src/inference/multi-model-network.js +5 -341
- package/src/inference/pipelines/audio/index.js +1 -20
- package/src/inference/pipelines/audio/mel.js +0 -38
- package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
- package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
- package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
- package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
- package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
- package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
- package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
- package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
- package/src/inference/pipelines/factory.js +14 -3
- package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +0 -6
- package/src/inference/pipelines/text/abort-contract.js +22 -0
- package/src/inference/pipelines/text/attention/attn-config.js +1 -16
- package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
- package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
- package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
- package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
- package/src/inference/pipelines/text/attention/interpreter.js +89 -439
- package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
- package/src/inference/pipelines/text/attention/plan.js +17 -0
- package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
- package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
- package/src/inference/pipelines/text/attention/projections.js +20 -537
- package/src/inference/pipelines/text/attention/query-transform.js +11 -0
- package/src/inference/pipelines/text/attention/record.js +0 -6
- package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
- package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
- package/src/inference/pipelines/text/attention/run.js +0 -6
- package/src/inference/pipelines/text/attention/types.js +3 -31
- package/src/inference/pipelines/text/chat-format.d.ts +3 -0
- package/src/inference/pipelines/text/chat-format.js +42 -0
- package/src/inference/pipelines/text/config/normalization.js +729 -0
- package/src/inference/pipelines/text/config/validation.js +9 -0
- package/src/inference/pipelines/text/config.d.ts +12 -1
- package/src/inference/pipelines/text/config.js +14 -1228
- package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
- package/src/inference/pipelines/text/debug-utils/index.js +3 -8
- package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
- package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
- package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +1 -68
- package/src/inference/pipelines/text/embed.js +178 -241
- package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
- package/src/inference/pipelines/text/embedding-contract.js +21 -0
- package/src/inference/pipelines/text/embedding-extraction.js +225 -0
- package/src/inference/pipelines/text/embedding-normalization.js +39 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
- package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
- package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
- package/src/inference/pipelines/text/execution-v1.js +7 -283
- package/src/inference/pipelines/text/execution.js +517 -0
- package/src/inference/pipelines/text/ffn/dense-executor.js +718 -0
- package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
- package/src/inference/pipelines/text/ffn/dense.js +4 -968
- package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
- package/src/inference/pipelines/text/ffn/standard.js +2 -1
- package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
- package/src/inference/pipelines/text/generator/decode-runtime.js +991 -0
- package/src/inference/pipelines/text/generator/decode.js +952 -0
- package/src/inference/pipelines/text/generator/diffusion.js +607 -0
- package/src/inference/pipelines/text/generator/logits-config.js +63 -0
- package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
- package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
- package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
- package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
- package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
- package/src/inference/pipelines/text/generator/recovery.js +310 -0
- package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
- package/src/inference/pipelines/text/generator/text.js +448 -0
- package/src/inference/pipelines/text/generator/training.js +752 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
- package/src/inference/pipelines/text/generator-logits-observation.js +45 -0
- package/src/inference/pipelines/text/generator-runtime.js +5 -135
- package/src/inference/pipelines/text/generator-steps.js +30 -972
- package/src/inference/pipelines/text/generator.js +165 -3544
- package/src/inference/pipelines/text/image-transcription.js +304 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
- package/src/inference/pipelines/text/init-rope.js +37 -189
- package/src/inference/pipelines/text/init.d.ts +8 -3
- package/src/inference/pipelines/text/init.js +6 -216
- package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
- package/src/inference/pipelines/text/layer-input-execution.js +435 -0
- package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
- package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
- package/src/inference/pipelines/text/layer-plan.js +45 -3
- package/src/inference/pipelines/text/layer.js +52 -522
- package/src/inference/pipelines/text/lifecycle.js +839 -0
- package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
- package/src/inference/pipelines/text/linear-attention.js +7 -537
- package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
- package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
- package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
- package/src/inference/pipelines/text/logits/cpu.js +8 -95
- package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
- package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
- package/src/inference/pipelines/text/logits/gpu.js +116 -579
- package/src/inference/pipelines/text/logits/index.d.ts +2 -1
- package/src/inference/pipelines/text/logits/index.js +62 -102
- package/src/inference/pipelines/text/logits/output-transform.js +47 -0
- package/src/inference/pipelines/text/logits/plan.js +184 -0
- package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
- package/src/inference/pipelines/text/logits/readback.js +9 -0
- package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/lora-apply.js +12 -4
- package/src/inference/pipelines/text/lora-types.d.ts +12 -0
- package/src/inference/pipelines/text/modality-token-contract.js +115 -0
- package/src/inference/pipelines/text/model-load.js +0 -20
- package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
- package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
- package/src/inference/pipelines/text/moe/plan.js +47 -0
- package/src/inference/pipelines/text/moe-cpu.js +9 -135
- package/src/inference/pipelines/text/moe-gpu.js +21 -545
- package/src/inference/pipelines/text/normalization-contract.js +17 -0
- package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
- package/src/inference/pipelines/text/per-layer/plan.js +362 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
- package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
- package/src/inference/pipelines/text/probes.js +20 -0
- package/src/inference/pipelines/text/state.d.ts +8 -0
- package/src/inference/pipelines/text/state.js +1 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
- package/src/inference/pipelines/text/types.d.ts +4 -0
- package/src/inference/pipelines/text/weight-loading-config.js +194 -0
- package/src/inference/pipelines/text.d.ts +2 -2
- package/src/inference/pipelines/text.js +46 -1633
- package/src/inference/pipelines/video/frame-sampler.js +0 -13
- package/src/inference/pipelines/video/index.js +0 -21
- package/src/inference/pipelines/vision/attention.js +59 -0
- package/src/inference/pipelines/vision/encoder.js +254 -369
- package/src/inference/pipelines/vision/gemma4.js +204 -318
- package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
- package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
- package/src/inference/pipelines/vision/glmocr.js +557 -0
- package/src/inference/pipelines/vision/image-preprocess.js +0 -27
- package/src/inference/pipelines/vision/index.js +8 -27
- package/src/inference/pipelines/vision/ops.js +11 -33
- package/src/inference/pipelines/vision/patch-embed.js +51 -145
- package/src/inference/pipelines/vision/spatial-merge.js +78 -0
- package/src/inference/pipelines/vision/weight-loading.js +79 -0
- package/src/inference/refactor-receipt.js +2 -2
- package/src/inference/rerank.js +182 -0
- package/src/inference/tokenizer.js +1 -1
- package/src/inference/tokenizers/behavior-flags.js +6 -3
- package/src/inference/tokenizers/bundled/decoders.js +133 -0
- package/src/inference/tokenizers/bundled/registry.js +216 -0
- package/src/inference/tokenizers/bundled.js +23 -353
- package/src/loader/doppler-loader.d.ts +4 -0
- package/src/loader/doppler-loader.js +119 -930
- package/src/loader/embedding-limit-preflight.js +11 -4
- package/src/loader/embedding-loader.js +1 -3
- package/src/loader/gpu-tensor-materialization.js +20 -0
- package/src/loader/layer-loader.js +15 -119
- package/src/loader/layer-tensor-aliases.js +99 -0
- package/src/loader/layer-weight-shape.js +29 -0
- package/src/loader/load-timing.js +64 -0
- package/src/loader/loader-types.d.ts +3 -0
- package/src/loader/model-executor.js +819 -0
- package/src/loader/model-load-validation.js +15 -0
- package/src/loader/quantization-constants.d.ts +11 -29
- package/src/loader/quantization-constants.js +1 -2
- package/src/loader/shard-resolver.js +0 -14
- package/src/loader/tensor-load-error.js +14 -0
- package/src/loader/tensor-shard-indices.js +8 -0
- package/src/loader/tensors/bf16-loader.js +125 -0
- package/src/loader/tensors/functional-descriptor-loader.js +3 -375
- package/src/loader/tensors/load-executor.js +577 -0
- package/src/loader/tensors/source-transform/executor.js +123 -0
- package/src/loader/tensors/source-transform/plan.js +107 -0
- package/src/loader/tensors/source-transform.js +15 -264
- package/src/loader/tensors/tensor-loader.d.ts +4 -4
- package/src/loader/tensors/tensor-loader.js +7 -653
- package/src/memory/buffer-pool.js +7 -29
- package/src/rules/capability-transform-resolver.js +80 -0
- package/src/rules/execution-contract.d.ts +73 -0
- package/src/rules/execution-contract.js +429 -0
- package/src/rules/execution-rules-contract-check.js +1 -1
- package/src/rules/generated/rule-bundle.json +4305 -0
- package/src/rules/inference/capability-transforms.rules.json +3 -3
- package/src/rules/layer-pattern-contract-check.js +1 -1
- package/src/rules/rule-registry.js +70 -57
- package/src/storage/artifact-storage-context.js +2 -2
- package/src/storage/backend-selection.js +43 -0
- package/src/storage/distribution-transport.js +5 -0
- package/src/storage/download/integrity.d.ts +33 -0
- package/src/storage/download/integrity.js +130 -0
- package/src/storage/download/resume-inspection.d.ts +22 -0
- package/src/storage/download/resume-inspection.js +134 -0
- package/src/storage/download/retry.js +57 -0
- package/src/storage/download/state.js +131 -0
- package/src/storage/download/transport.js +99 -0
- package/src/storage/downloader.js +74 -453
- package/src/storage/model-cache.d.ts +1 -0
- package/src/storage/model-cache.js +51 -11
- package/src/storage/shard-manager.d.ts +5 -0
- package/src/storage/shard-manager.js +50 -369
- package/src/storage/shards/index.js +60 -0
- package/src/storage/shards/integrity.js +225 -0
- package/src/storage/shards/lifecycle.js +22 -0
- package/src/storage/source-artifact-store.js +14 -4
- package/src/tooling/boundary-evidence.d.ts +4 -4
- package/src/tooling/boundary-evidence.js +20 -20
- package/src/tooling/browser-command-runner.js +13 -6
- package/src/tooling/capsule-v2.d.ts +11 -0
- package/src/tooling/capsule-v2.js +58 -0
- package/src/tooling/capsule.js +20 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.js +163 -5
- package/src/tooling/command-api.d.ts +42 -3
- package/src/tooling/command-api.js +7 -2
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
- package/src/tooling/command-runner.html +49 -1
- package/src/tooling/command-validation.js +1 -24
- package/src/tooling/conversion-run-timing.js +24 -0
- package/src/tooling/diagnose-runner.js +11 -9
- package/src/tooling/distribution-shard-transport.js +11 -0
- package/src/tooling/execution-cost-ledger.d.ts +76 -2
- package/src/tooling/execution-cost-ledger.js +197 -15
- package/src/tooling/hf-registry/catalog-io.js +63 -0
- package/src/tooling/litert/package.js +863 -0
- package/src/tooling/litert-package-runtime.js +3 -849
- package/src/tooling/model-capsule-forge.js +478 -0
- package/src/tooling/node-browser/execution.d.ts +40 -0
- package/src/tooling/node-browser/execution.js +308 -0
- package/src/tooling/node-browser/static-server.js +188 -0
- package/src/tooling/node-browser/transport.d.ts +80 -0
- package/src/tooling/node-browser/transport.js +622 -0
- package/src/tooling/node-browser-command-runner.d.ts +18 -78
- package/src/tooling/node-browser-command-runner.js +10 -1112
- package/src/tooling/node-capsule-artifact-store.js +44 -0
- package/src/tooling/node-command-execution.js +237 -0
- package/src/tooling/node-command-runner.js +26 -205
- package/src/tooling/node-converter/input.js +521 -0
- package/src/tooling/node-converter/output.js +244 -0
- package/src/tooling/node-converter-chunk-batches.js +21 -0
- package/src/tooling/node-converter.js +7 -720
- package/src/tooling/node-source-runtime.js +16 -276
- package/src/tooling/node-webgpu-provider.v1.json +33 -0
- package/src/tooling/node-webgpu.js +178 -453
- package/src/tooling/opfs-cache.d.ts +5 -0
- package/src/tooling/opfs-cache.js +4 -0
- package/src/tooling/production-release.d.ts +19 -0
- package/src/tooling/production-release.js +647 -0
- package/src/tooling/program-bundle/embedding-reference.js +19 -0
- package/src/tooling/program-bundle/materialize.js +324 -0
- package/src/tooling/program-bundle/rerank-reference.js +22 -0
- package/src/tooling/program-bundle/sequence-reference.js +42 -0
- package/src/tooling/program-bundle/validation.js +31 -0
- package/src/tooling/program-bundle/wgsl-closure.js +254 -0
- package/src/tooling/program-bundle-parity.d.ts +11 -4
- package/src/tooling/program-bundle-parity.js +262 -109
- package/src/tooling/program-bundle.d.ts +7 -0
- package/src/tooling/program-bundle.js +186 -591
- package/src/tooling/provider-v1-contract.d.ts +61 -0
- package/src/tooling/provider-v1-contract.js +192 -0
- package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
- package/src/tooling/registered-variant-calibration-job.js +210 -0
- package/src/tooling/registered-variant-calibration.d.ts +167 -7
- package/src/tooling/registered-variant-calibration.js +57 -7
- package/src/tooling/runtime-optimization/candidates.js +664 -0
- package/src/tooling/runtime-optimization-campaign.js +166 -0
- package/src/tooling/runtime-optimization-index.js +16 -1
- package/src/tooling/runtime-optimization.d.ts +44 -0
- package/src/tooling/runtime-optimization.js +13 -656
- package/src/tooling/source-intake.d.ts +1 -1
- package/src/tooling/source-intake.js +34 -17
- package/src/tooling/source-package-profiles.js +1 -1
- package/src/tooling/source-runtime/bundle-materialize.js +298 -0
- package/src/tooling/source-runtime/bundle-plan.js +121 -0
- package/src/tooling/source-runtime/node-loader.js +258 -0
- package/src/tooling/source-runtime-bundle.js +26 -411
- package/src/tooling/source-runtime-converter-config.js +1 -1
- package/src/tooling-experimental-exports.js +0 -5
- package/src/tooling-exports/evidence.d.ts +14 -3
- package/src/tooling-exports/evidence.js +14 -3
- package/src/tooling-exports/storage.d.ts +5 -0
- package/src/tooling-exports/storage.js +4 -0
- package/src/tooling-exports.d.ts +3 -0
- package/src/tooling-exports.js +2 -0
- package/src/tooling-exports.shared.d.ts +17 -3
- package/src/tooling-exports.shared.js +9 -3
- package/src/training.d.ts +6 -6
- package/src/training.js +4 -4
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +512 -0
- package/tests/kernels/browser/test-page.js +169 -14
- package/tests/kernels/harness/tolerance.js +1 -0
- package/tests/kernels/reference/index.js +13 -1
- package/tests/kernels/reference/vision.js +160 -0
- package/src/converter/quantizer.d.ts +0 -121
- package/src/gpu/kernels/backward/utils.d.ts +0 -90
- package/src/gpu/kernels/uniform-utils.d.ts +0 -66
- package/src/gpu/kernels/utils.d.ts +0 -147
- package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
- package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
- package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
- package/src/inference/pipelines/text/moe-helpers.js +0 -21
- package/src/rules/converter/execution.rules.json +0 -20
- package/src/rules/converter/tensor-roles.rules.json +0 -13
- package/src/rules/converter/tokenizer.rules.json +0 -12
- package/src/rules/inference/attention.rules.json +0 -69
- package/src/rules/inference/config.rules.json +0 -82
- package/src/rules/inference/dtype.rules.json +0 -113
- package/src/rules/inference/execution.rules.json +0 -195
- package/src/rules/inference/ffn.rules.json +0 -123
- package/src/rules/inference/layer-pattern.rules.json +0 -16
- package/src/rules/inference/layer.rules.json +0 -7
- package/src/rules/inference/moe.rules.json +0 -145
- package/src/rules/kernels/attention.rules.json +0 -68
- package/src/rules/kernels/conv2d.rules.json +0 -6
- package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
- package/src/rules/kernels/dequant.rules.json +0 -58
- package/src/rules/kernels/energy.rules.json +0 -22
- package/src/rules/kernels/fused-ffn.rules.json +0 -15
- package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
- package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
- package/src/rules/kernels/gather.rules.json +0 -93
- package/src/rules/kernels/gelu.rules.json +0 -11
- package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
- package/src/rules/kernels/groupnorm.rules.json +0 -10
- package/src/rules/kernels/kv_quantize.rules.json +0 -9
- package/src/rules/kernels/layernorm.rules.json +0 -6
- package/src/rules/kernels/linear-attention.rules.json +0 -6
- package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
- package/src/rules/kernels/matmul.rules.json +0 -95
- package/src/rules/kernels/modulate.rules.json +0 -6
- package/src/rules/kernels/moe.rules.gptoss.json +0 -105
- package/src/rules/kernels/moe.rules.json +0 -14
- package/src/rules/kernels/moe.rules.mixtral.json +0 -75
- package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
- package/src/rules/kernels/relu.rules.json +0 -6
- package/src/rules/kernels/rep-penalty.rules.json +0 -6
- package/src/rules/kernels/repeat-channels.rules.json +0 -6
- package/src/rules/kernels/residual.rules.json +0 -12
- package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
- package/src/rules/kernels/rmsnorm.rules.json +0 -46
- package/src/rules/kernels/rope-qk.rules.json +0 -6
- package/src/rules/kernels/rope.rules.json +0 -6
- package/src/rules/kernels/sample.rules.json +0 -6
- package/src/rules/kernels/scale.rules.json +0 -8
- package/src/rules/kernels/silu.rules.json +0 -22
- package/src/rules/kernels/softmax.rules.json +0 -25
- package/src/rules/kernels/split-qg.rules.json +0 -6
- package/src/rules/kernels/split-qkv.rules.json +0 -6
- package/src/rules/kernels/upsample2d.rules.json +0 -6
- package/src/rules/loader/tensor-loader.rules.json +0 -42
- package/src/rules/loader/weights.rules.json +0 -42
- package/src/rules/tooling/command-runtime.rules.json +0 -65
- package/src/tooling/hf-registry-utils.js +0 -395
- /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
- /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
- /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
- /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
- /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
- /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
- /package/src/{utils → formats}/array-buffer.js +0 -0
- /package/src/{utils → formats}/canonical-hash.js +0 -0
- /package/src/{utils → formats}/clone-json.js +0 -0
- /package/src/{utils → formats}/encode-utf8.js +0 -0
- /package/src/{utils → formats}/load-json.js +0 -0
- /package/src/{utils → formats}/plain-object.js +0 -0
- /package/src/{utils → formats}/sha256.js +0 -0
- /package/src/{utils → formats}/stable-sort-object.js +0 -0
- /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
- /package/src/{utils → storage}/hf-resolve-url.js +0 -0
- /package/src/{utils → storage}/runtime-env.js +0 -0
- /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
|
@@ -1,4 +1,10 @@
|
|
|
1
|
-
|
|
1
|
+
|
|
2
|
+
import { F16_TO_F32_ACTIVATION_MAP, F32_TO_F16_ACTIVATION_MAP, KERNEL_FILE_PRECISION_PATCHES, cloneGraph, deriveF16AttentionKernelEntry, deriveKernelEntry, deriveKernelEntryWithPrecision, deriveKernelKey, deriveLinearDecodeF16KernelEntry, deriveLmHeadDecodeF16KernelEntry, deriveQ4DecodeF16KernelEntry, deriveQ4PrefillF16AccumKernelEntry, deriveQ4PrefillF16KernelEntry, deriveQ4WideTilePrefillF16KernelEntry, findPhaseStep, narrowToF16Activations, replacePhaseStepKernelKey, useQwen36F16Activations, useQwenDecodeF16Matmuls, useQwenF16PrimaryMatmuls } from './execution-graph/dtype.js';
|
|
3
|
+
import { LAYER_PROJECTION_OPS, useGemma4Int4PleAf16Activations, useGemma4Int4PleSelectiveF16Decode, useGemma4TextF16ActivationsForLane } from './execution-graph/fusion.js';
|
|
4
|
+
import { deriveMappedKernelEntry, deriveQ4DecodeF32ActivationKernelEntry, deriveQ4PrefillF32ActivationKernelEntry, findKernelKeyByFileAndEntry, hasKernelFile, remapDenseQ4KPrefillToQ4Native, remapStepKeys, removeSubgroups, widenProjectionWeightsToF32, widenToF32Activations } from './execution-graph/validation.js';
|
|
5
|
+
export { remapDenseQ4KPrefillToQ4Native, removeSubgroups, widenProjectionWeightsToF32, widenToF32Activations } from './execution-graph/validation.js';
|
|
6
|
+
export { useGemma4Int4PleAf16Activations, useGemma4Int4PleSelectiveF16Decode } from './execution-graph/fusion.js';
|
|
7
|
+
export { narrowToF16Activations, useQwen36F16Activations, useQwenDecodeF16Matmuls, useQwenF16PrimaryMatmuls } from './execution-graph/dtype.js';// =============================================================================
|
|
2
8
|
// Execution Graph Transforms
|
|
3
9
|
// =============================================================================
|
|
4
10
|
//
|
|
@@ -16,38 +22,11 @@
|
|
|
16
22
|
/*
|
|
17
23
|
* Deep-clone an execution graph.
|
|
18
24
|
*/
|
|
19
|
-
function cloneGraph(graph) {
|
|
20
|
-
return structuredClone(graph);
|
|
21
|
-
}
|
|
22
25
|
|
|
23
26
|
/*
|
|
24
27
|
* Shader files that require subgroups even though "subgroup" is not in the filename.
|
|
25
28
|
* Online attention kernels use subgroup reductions internally.
|
|
26
29
|
*/
|
|
27
|
-
const SUBGROUP_REQUIRING_FILES = new Set([
|
|
28
|
-
'attention_decode_online_f16kv.wgsl',
|
|
29
|
-
'attention_decode_online_f16.wgsl',
|
|
30
|
-
]);
|
|
31
|
-
|
|
32
|
-
const KERNEL_FILE_PRECISION_PATCHES = new Map([
|
|
33
|
-
['matmul_gemv_subgroup_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
34
|
-
['matmul_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
35
|
-
['matmul_f16_tiled.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
36
|
-
['silu_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
37
|
-
['fused_matmul_q4_multicol_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
38
|
-
['fused_matmul_q4_multicol_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
39
|
-
['fused_matmul_q4_batched_f16acc_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
40
|
-
['fused_matmul_q4.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
41
|
-
['fused_matmul_q4_batched.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
42
|
-
['fused_matmul_q4_batched_multicol_shared.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
43
|
-
['fused_matmul_q4_widetile_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
44
|
-
['matmul_gemv_subgroup.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
45
|
-
['matmul_f16w_f32a.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
46
|
-
['matmul_f16w_f32a_tiled.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
47
|
-
['matmul_f32.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
48
|
-
['gather_f16_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
49
|
-
['gather_f16_vec4_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
50
|
-
]);
|
|
51
30
|
|
|
52
31
|
export function getKernelFilePrecisionPatch(kernel) {
|
|
53
32
|
return KERNEL_FILE_PRECISION_PATCHES.get(kernel) ?? null;
|
|
@@ -56,114 +35,26 @@ export function getKernelFilePrecisionPatch(kernel) {
|
|
|
56
35
|
/*
|
|
57
36
|
* Check whether a kernel entry requires subgroup support.
|
|
58
37
|
*/
|
|
59
|
-
function isSubgroupKernel(kernelEntry) {
|
|
60
|
-
if (typeof kernelEntry.kernel !== 'string') return false;
|
|
61
|
-
return kernelEntry.kernel.includes('subgroup') || SUBGROUP_REQUIRING_FILES.has(kernelEntry.kernel);
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
function requiresNoSubgroupFallback(kernelEntry) {
|
|
65
|
-
if (typeof kernelEntry?.kernel !== 'string') return false;
|
|
66
|
-
return isSubgroupKernel(kernelEntry) || kernelEntry.kernel.startsWith('fused_matmul_q4');
|
|
67
|
-
}
|
|
68
38
|
|
|
69
39
|
/*
|
|
70
40
|
* Find all kernel keys in the graph whose `kernel` file matches the given filename.
|
|
71
41
|
*/
|
|
72
|
-
function findKernelKeysByFile(graph, filename) {
|
|
73
|
-
const keys = [];
|
|
74
|
-
for (const [key, entry] of Object.entries(graph.kernels)) {
|
|
75
|
-
if (entry.kernel === filename) {
|
|
76
|
-
keys.push(key);
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
return keys;
|
|
80
|
-
}
|
|
81
42
|
|
|
82
43
|
/*
|
|
83
44
|
* Check whether any kernel in the graph uses the given shader file.
|
|
84
45
|
*/
|
|
85
|
-
function hasKernelFile(graph, filename) {
|
|
86
|
-
return findKernelKeysByFile(graph, filename).length > 0;
|
|
87
|
-
}
|
|
88
46
|
|
|
89
47
|
/*
|
|
90
48
|
* Create a new kernel entry with the digest cleared (shader changed).
|
|
91
49
|
*/
|
|
92
|
-
function deriveKernelEntry(base, newFile, newEntry, constants) {
|
|
93
|
-
const derived = { ...base, kernel: newFile, entry: newEntry, digest: null };
|
|
94
|
-
if (constants === null) {
|
|
95
|
-
delete derived.constants;
|
|
96
|
-
} else if (constants !== undefined) {
|
|
97
|
-
derived.constants = { ...constants };
|
|
98
|
-
}
|
|
99
|
-
const precision = deriveKernelPrecision(base, newFile);
|
|
100
|
-
if (precision) {
|
|
101
|
-
derived.precision = precision;
|
|
102
|
-
} else {
|
|
103
|
-
delete derived.precision;
|
|
104
|
-
}
|
|
105
|
-
return derived;
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
function deriveMappedKernelEntry(base, newFile) {
|
|
109
|
-
const newEntry = newFile === 'matmul_f32.wgsl' ? 'main' : base.entry;
|
|
110
|
-
return deriveKernelEntry(base, newFile, newEntry);
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
function deriveKernelPrecision(base, newFile) {
|
|
114
|
-
const precision = base.precision ? { ...base.precision } : {};
|
|
115
|
-
const precisionPatch = KERNEL_FILE_PRECISION_PATCHES.get(newFile);
|
|
116
|
-
if (precisionPatch) {
|
|
117
|
-
Object.assign(precision, precisionPatch);
|
|
118
|
-
}
|
|
119
|
-
if (!String(newFile).startsWith('attention')) {
|
|
120
|
-
return Object.keys(precision).length > 0 ? precision : null;
|
|
121
|
-
}
|
|
122
|
-
if (newFile.includes('_f16kv')) {
|
|
123
|
-
precision.activationDtype = 'f32';
|
|
124
|
-
precision.kvDtype = 'f16';
|
|
125
|
-
return precision;
|
|
126
|
-
}
|
|
127
|
-
if (newFile.includes('_f16')) {
|
|
128
|
-
precision.activationDtype = 'f16';
|
|
129
|
-
precision.kvDtype = 'f16';
|
|
130
|
-
return precision;
|
|
131
|
-
}
|
|
132
|
-
precision.activationDtype = 'f32';
|
|
133
|
-
precision.kvDtype = 'f32';
|
|
134
|
-
return precision;
|
|
135
|
-
}
|
|
136
50
|
|
|
137
51
|
/*
|
|
138
52
|
* Derive a non-colliding kernel key name.
|
|
139
53
|
*/
|
|
140
|
-
function deriveKernelKey(kernels, baseKey, suffix) {
|
|
141
|
-
const candidate = `${baseKey}${suffix}`;
|
|
142
|
-
if (!kernels[candidate]) {
|
|
143
|
-
return candidate;
|
|
144
|
-
}
|
|
145
|
-
let counter = 2;
|
|
146
|
-
while (kernels[`${candidate}_${counter}`]) {
|
|
147
|
-
counter++;
|
|
148
|
-
}
|
|
149
|
-
return `${candidate}_${counter}`;
|
|
150
|
-
}
|
|
151
54
|
|
|
152
55
|
/*
|
|
153
56
|
* Replace kernel key references in step tuples.
|
|
154
57
|
*/
|
|
155
|
-
function remapStepKeys(steps, keyMap) {
|
|
156
|
-
return steps.map((step) => {
|
|
157
|
-
const kernelKey = step[1];
|
|
158
|
-
const replacement = keyMap.get(kernelKey);
|
|
159
|
-
if (replacement !== undefined) {
|
|
160
|
-
const newStep = [...step];
|
|
161
|
-
newStep[1] = replacement;
|
|
162
|
-
return newStep;
|
|
163
|
-
}
|
|
164
|
-
return step;
|
|
165
|
-
});
|
|
166
|
-
}
|
|
167
58
|
|
|
168
59
|
/*
|
|
169
60
|
* Check whether a step tuple's kernel key resolves to the given shader file.
|
|
@@ -178,30 +69,10 @@ function stepUsesFile(graph, step, filename) {
|
|
|
178
69
|
* Find the first kernel key used by matching ops in a phase whose shader file
|
|
179
70
|
* satisfies the provided predicate.
|
|
180
71
|
*/
|
|
181
|
-
function findPhaseKernelKey(graph, steps, ops, predicate) {
|
|
182
|
-
for (const step of steps || []) {
|
|
183
|
-
if (!ops.has(step[0])) {
|
|
184
|
-
continue;
|
|
185
|
-
}
|
|
186
|
-
const entry = graph.kernels[step[1]];
|
|
187
|
-
if (entry && predicate(entry)) {
|
|
188
|
-
return step[1];
|
|
189
|
-
}
|
|
190
|
-
}
|
|
191
|
-
return null;
|
|
192
|
-
}
|
|
193
72
|
|
|
194
73
|
/*
|
|
195
74
|
* Find an existing kernel key by shader file and entry point.
|
|
196
75
|
*/
|
|
197
|
-
function findKernelKeyByFileAndEntry(graph, filename, entryPoint) {
|
|
198
|
-
for (const [key, entry] of Object.entries(graph.kernels)) {
|
|
199
|
-
if (entry.kernel === filename && entry.entry === entryPoint) {
|
|
200
|
-
return key;
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
return null;
|
|
204
|
-
}
|
|
205
76
|
|
|
206
77
|
function normalizeLayerType(layerType) {
|
|
207
78
|
return typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
|
|
@@ -251,306 +122,6 @@ function buildGroupedLayerEntries(baseStep, targetLayers, replacementKernelKey)
|
|
|
251
122
|
return groupedEntries;
|
|
252
123
|
}
|
|
253
124
|
|
|
254
|
-
function replacePhaseStepKernelKey(steps, op, replacementKernelKey) {
|
|
255
|
-
if (!Array.isArray(steps) || steps.length === 0 || !replacementKernelKey) {
|
|
256
|
-
return { steps, changed: false };
|
|
257
|
-
}
|
|
258
|
-
let changed = false;
|
|
259
|
-
const nextSteps = steps.map((step) => {
|
|
260
|
-
if (!Array.isArray(step) || step[0] !== op) {
|
|
261
|
-
return step;
|
|
262
|
-
}
|
|
263
|
-
if (step[1] === replacementKernelKey) {
|
|
264
|
-
return step;
|
|
265
|
-
}
|
|
266
|
-
const replacement = [...step];
|
|
267
|
-
replacement[1] = replacementKernelKey;
|
|
268
|
-
changed = true;
|
|
269
|
-
return replacement;
|
|
270
|
-
});
|
|
271
|
-
return { steps: nextSteps, changed };
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
function findPhaseStep(steps, op) {
|
|
275
|
-
if (!Array.isArray(steps) || !op) {
|
|
276
|
-
return null;
|
|
277
|
-
}
|
|
278
|
-
for (const entry of steps) {
|
|
279
|
-
if (Array.isArray(entry)) {
|
|
280
|
-
if (entry[0] === op) {
|
|
281
|
-
return entry;
|
|
282
|
-
}
|
|
283
|
-
continue;
|
|
284
|
-
}
|
|
285
|
-
if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
|
|
286
|
-
continue;
|
|
287
|
-
}
|
|
288
|
-
const nested = findPhaseStep(entry.steps, op);
|
|
289
|
-
if (nested) {
|
|
290
|
-
return nested;
|
|
291
|
-
}
|
|
292
|
-
}
|
|
293
|
-
return null;
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
function deriveKernelEntryWithPrecision(base, precision) {
|
|
297
|
-
return {
|
|
298
|
-
...base,
|
|
299
|
-
precision: {
|
|
300
|
-
...(base.precision ?? {}),
|
|
301
|
-
...precision,
|
|
302
|
-
},
|
|
303
|
-
};
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
const ATTENTION_F16KV_TO_F16_MAP = new Map([
|
|
307
|
-
['attention_decode_online_f16kv.wgsl', 'attention_decode_online_f16.wgsl'],
|
|
308
|
-
['attention_decode_chunked_f16kv.wgsl', 'attention_decode_chunked_f16.wgsl'],
|
|
309
|
-
['attention_small_f16kv.wgsl', 'attention_small_f16.wgsl'],
|
|
310
|
-
['attention_streaming_f16kv.wgsl', 'attention_streaming_f16.wgsl'],
|
|
311
|
-
['attention_head512_f16kv.wgsl', 'attention_head512_f16.wgsl'],
|
|
312
|
-
]);
|
|
313
|
-
|
|
314
|
-
function deriveF16AttentionKernelEntry(base) {
|
|
315
|
-
if (typeof base?.kernel !== 'string') {
|
|
316
|
-
return null;
|
|
317
|
-
}
|
|
318
|
-
const replacement = ATTENTION_F16KV_TO_F16_MAP.get(base.kernel);
|
|
319
|
-
if (!replacement) {
|
|
320
|
-
return null;
|
|
321
|
-
}
|
|
322
|
-
return deriveKernelEntryWithPrecision(
|
|
323
|
-
deriveKernelEntry(base, replacement, base.entry),
|
|
324
|
-
{ activationDtype: 'f16', kvDtype: 'f16' }
|
|
325
|
-
);
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
function deriveLinearDecodeF16KernelEntry(base) {
|
|
329
|
-
const precision = {
|
|
330
|
-
inputDtype: 'f16',
|
|
331
|
-
outputDtype: 'f16',
|
|
332
|
-
};
|
|
333
|
-
if (base.kernel === 'fused_matmul_q4.wgsl' && base.entry === 'main_multicol') {
|
|
334
|
-
return {
|
|
335
|
-
...deriveKernelEntry(base, 'fused_matmul_q4_multicol_f16a.wgsl', 'main_multicol_f16a'),
|
|
336
|
-
precision: {
|
|
337
|
-
...(base.precision ?? {}),
|
|
338
|
-
...precision,
|
|
339
|
-
},
|
|
340
|
-
};
|
|
341
|
-
}
|
|
342
|
-
if (
|
|
343
|
-
(base.kernel === 'fused_matmul_q4_multicol_f16.wgsl' && base.entry === 'main_multicol_f16')
|
|
344
|
-
|| (base.kernel === 'fused_matmul_q4_multicol_f16a.wgsl' && base.entry === 'main_multicol_f16a')
|
|
345
|
-
) {
|
|
346
|
-
return deriveKernelEntryWithPrecision(base, precision);
|
|
347
|
-
}
|
|
348
|
-
return null;
|
|
349
|
-
}
|
|
350
|
-
|
|
351
|
-
function deriveLmHeadDecodeF16KernelEntry(base) {
|
|
352
|
-
const precision = {
|
|
353
|
-
inputDtype: 'f16',
|
|
354
|
-
outputDtype: 'f16',
|
|
355
|
-
};
|
|
356
|
-
if (base.kernel === 'matmul_gemv_subgroup.wgsl' && base.entry === 'main_multicol') {
|
|
357
|
-
return {
|
|
358
|
-
...deriveKernelEntry(base, 'matmul_gemv_subgroup_f16a.wgsl', 'main_multicol'),
|
|
359
|
-
precision: {
|
|
360
|
-
...(base.precision ?? {}),
|
|
361
|
-
...precision,
|
|
362
|
-
},
|
|
363
|
-
};
|
|
364
|
-
}
|
|
365
|
-
if (base.kernel === 'matmul_gemv_subgroup_f16a.wgsl' && base.entry === 'main_multicol') {
|
|
366
|
-
return deriveKernelEntryWithPrecision(base, precision);
|
|
367
|
-
}
|
|
368
|
-
return null;
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
function deriveDenseDecodeF16KernelEntry(base) {
|
|
372
|
-
if (typeof base?.kernel !== 'string') {
|
|
373
|
-
return null;
|
|
374
|
-
}
|
|
375
|
-
const precision = {
|
|
376
|
-
inputDtype: 'f16',
|
|
377
|
-
outputDtype: 'f16',
|
|
378
|
-
};
|
|
379
|
-
if (base.kernel === 'matmul_gemv_subgroup.wgsl') {
|
|
380
|
-
return {
|
|
381
|
-
...deriveKernelEntry(base, 'matmul_gemv_subgroup_f16a.wgsl', base.entry ?? 'main'),
|
|
382
|
-
precision: {
|
|
383
|
-
...(base.precision ?? {}),
|
|
384
|
-
...precision,
|
|
385
|
-
},
|
|
386
|
-
};
|
|
387
|
-
}
|
|
388
|
-
if (base.kernel === 'matmul_gemv_subgroup_f16a.wgsl') {
|
|
389
|
-
return deriveKernelEntryWithPrecision(base, precision);
|
|
390
|
-
}
|
|
391
|
-
return null;
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
function deriveQ4DecodeF16KernelEntry(base) {
|
|
395
|
-
if (typeof base?.kernel !== 'string') {
|
|
396
|
-
return null;
|
|
397
|
-
}
|
|
398
|
-
const precision = {
|
|
399
|
-
inputDtype: 'f16',
|
|
400
|
-
outputDtype: 'f16',
|
|
401
|
-
};
|
|
402
|
-
if (base.kernel === 'fused_matmul_q4.wgsl') {
|
|
403
|
-
return {
|
|
404
|
-
...deriveKernelEntry(base, 'fused_matmul_q4_multicol_f16a.wgsl', 'main_multicol_f16a', null),
|
|
405
|
-
precision: {
|
|
406
|
-
...(base.precision ?? {}),
|
|
407
|
-
...precision,
|
|
408
|
-
},
|
|
409
|
-
};
|
|
410
|
-
}
|
|
411
|
-
if (
|
|
412
|
-
base.kernel === 'fused_matmul_q4_multicol_f16.wgsl'
|
|
413
|
-
|| base.kernel === 'fused_matmul_q4_multicol_f16a.wgsl'
|
|
414
|
-
) {
|
|
415
|
-
return deriveKernelEntryWithPrecision(base, precision);
|
|
416
|
-
}
|
|
417
|
-
return null;
|
|
418
|
-
}
|
|
419
|
-
|
|
420
|
-
function deriveQ4PrefillF16KernelEntry(base) {
|
|
421
|
-
if (typeof base?.kernel !== 'string') {
|
|
422
|
-
return null;
|
|
423
|
-
}
|
|
424
|
-
const precision = {
|
|
425
|
-
inputDtype: 'f16',
|
|
426
|
-
outputDtype: 'f16',
|
|
427
|
-
};
|
|
428
|
-
if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
|
|
429
|
-
return deriveKernelEntryWithPrecision(base, precision);
|
|
430
|
-
}
|
|
431
|
-
if (base.kernel.startsWith('fused_matmul_q4_batched')) {
|
|
432
|
-
return {
|
|
433
|
-
...deriveKernelEntry(base, 'fused_matmul_q4_batched_f16a.wgsl', 'main_batched_f16a', null),
|
|
434
|
-
precision: {
|
|
435
|
-
...(base.precision ?? {}),
|
|
436
|
-
...precision,
|
|
437
|
-
},
|
|
438
|
-
};
|
|
439
|
-
}
|
|
440
|
-
return null;
|
|
441
|
-
}
|
|
442
|
-
|
|
443
|
-
function deriveQ4WideTilePrefillF16KernelEntry(base) {
|
|
444
|
-
if (typeof base?.kernel !== 'string') {
|
|
445
|
-
return null;
|
|
446
|
-
}
|
|
447
|
-
const precision = {
|
|
448
|
-
inputDtype: 'f16',
|
|
449
|
-
outputDtype: 'f16',
|
|
450
|
-
};
|
|
451
|
-
if (base.kernel === 'fused_matmul_q4_widetile.wgsl') {
|
|
452
|
-
return {
|
|
453
|
-
...deriveKernelEntry(base, 'fused_matmul_q4_widetile_f16a.wgsl', 'main', null),
|
|
454
|
-
precision: {
|
|
455
|
-
...(base.precision ?? {}),
|
|
456
|
-
...precision,
|
|
457
|
-
},
|
|
458
|
-
};
|
|
459
|
-
}
|
|
460
|
-
if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
|
|
461
|
-
return deriveKernelEntryWithPrecision(base, precision);
|
|
462
|
-
}
|
|
463
|
-
return null;
|
|
464
|
-
}
|
|
465
|
-
|
|
466
|
-
function deriveQ4PrefillF16AccumKernelEntry(base) {
|
|
467
|
-
if (typeof base?.kernel !== 'string') {
|
|
468
|
-
return null;
|
|
469
|
-
}
|
|
470
|
-
const precision = {
|
|
471
|
-
inputDtype: 'f16',
|
|
472
|
-
outputDtype: 'f16',
|
|
473
|
-
};
|
|
474
|
-
if (
|
|
475
|
-
base.kernel === 'fused_matmul_q4_widetile.wgsl'
|
|
476
|
-
|| base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl'
|
|
477
|
-
|| base.kernel.startsWith('fused_matmul_q4_batched')
|
|
478
|
-
) {
|
|
479
|
-
return {
|
|
480
|
-
...deriveKernelEntry(base, 'fused_matmul_q4_batched_f16acc_f16a.wgsl', 'main_batched_f16acc_f16a', null),
|
|
481
|
-
precision: {
|
|
482
|
-
...(base.precision ?? {}),
|
|
483
|
-
...precision,
|
|
484
|
-
},
|
|
485
|
-
};
|
|
486
|
-
}
|
|
487
|
-
return null;
|
|
488
|
-
}
|
|
489
|
-
|
|
490
|
-
function deriveQ4DecodeF32ActivationKernelEntry(base) {
|
|
491
|
-
if (typeof base?.kernel !== 'string') {
|
|
492
|
-
return null;
|
|
493
|
-
}
|
|
494
|
-
const precision = {
|
|
495
|
-
inputDtype: 'f32',
|
|
496
|
-
outputDtype: 'f32',
|
|
497
|
-
};
|
|
498
|
-
if (
|
|
499
|
-
base.kernel === 'fused_matmul_q4_multicol_f16.wgsl'
|
|
500
|
-
|| base.kernel === 'fused_matmul_q4_multicol_f16a.wgsl'
|
|
501
|
-
) {
|
|
502
|
-
return deriveKernelEntryWithPrecision(
|
|
503
|
-
deriveKernelEntry(base, 'fused_matmul_q4.wgsl', 'main_multicol', null),
|
|
504
|
-
precision
|
|
505
|
-
);
|
|
506
|
-
}
|
|
507
|
-
if (base.kernel === 'fused_matmul_q4_f16a.wgsl') {
|
|
508
|
-
return deriveKernelEntryWithPrecision(
|
|
509
|
-
deriveKernelEntry(base, 'fused_matmul_q4.wgsl', 'main', null),
|
|
510
|
-
precision
|
|
511
|
-
);
|
|
512
|
-
}
|
|
513
|
-
return null;
|
|
514
|
-
}
|
|
515
|
-
|
|
516
|
-
function deriveQ4PrefillF32ActivationKernelEntry(base, options = {}) {
|
|
517
|
-
if (typeof base?.kernel !== 'string') {
|
|
518
|
-
return null;
|
|
519
|
-
}
|
|
520
|
-
const replacement = options.fullF32 === true
|
|
521
|
-
? {
|
|
522
|
-
kernel: 'fused_matmul_q4_batched_multicol_shared.wgsl',
|
|
523
|
-
entry: 'main',
|
|
524
|
-
}
|
|
525
|
-
: {
|
|
526
|
-
kernel: 'fused_matmul_q4_widetile.wgsl',
|
|
527
|
-
entry: 'main',
|
|
528
|
-
};
|
|
529
|
-
if (base.kernel === 'fused_matmul_q4_widetile.wgsl' && options.fullF32 === true) {
|
|
530
|
-
return deriveKernelEntryWithPrecision(
|
|
531
|
-
deriveKernelEntry(base, replacement.kernel, replacement.entry, null),
|
|
532
|
-
{ inputDtype: 'f32', outputDtype: 'f32' }
|
|
533
|
-
);
|
|
534
|
-
}
|
|
535
|
-
if (
|
|
536
|
-
base.kernel !== 'fused_matmul_q4_batched_f16.wgsl'
|
|
537
|
-
&& base.kernel !== 'fused_matmul_q4_batched_f16a.wgsl'
|
|
538
|
-
&& base.kernel !== 'fused_matmul_q4_widetile_f16a.wgsl'
|
|
539
|
-
) {
|
|
540
|
-
return null;
|
|
541
|
-
}
|
|
542
|
-
if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
|
|
543
|
-
return deriveKernelEntryWithPrecision(
|
|
544
|
-
deriveKernelEntry(base, replacement.kernel, replacement.entry, null),
|
|
545
|
-
{ inputDtype: 'f32', outputDtype: 'f32' }
|
|
546
|
-
);
|
|
547
|
-
}
|
|
548
|
-
return deriveKernelEntryWithPrecision(
|
|
549
|
-
deriveKernelEntry(base, 'fused_matmul_q4_batched.wgsl', 'main_batched', null),
|
|
550
|
-
{ inputDtype: 'f32', outputDtype: 'f32' }
|
|
551
|
-
);
|
|
552
|
-
}
|
|
553
|
-
|
|
554
125
|
function replacePhaseStepEntries(steps, op, replacementEntries) {
|
|
555
126
|
if (!Array.isArray(steps) || steps.length === 0 || !Array.isArray(replacementEntries) || replacementEntries.length === 0) {
|
|
556
127
|
return { steps, changed: false };
|
|
@@ -580,86 +151,6 @@ function replacePhaseStepEntries(steps, op, replacementEntries) {
|
|
|
580
151
|
* Returns null if the graph has no subgroup kernels.
|
|
581
152
|
*
|
|
582
153
|
*/
|
|
583
|
-
export function removeSubgroups(graph, ctx) {
|
|
584
|
-
const hasAnyFallbackKernel = Object.values(graph.kernels).some(requiresNoSubgroupFallback);
|
|
585
|
-
if (!hasAnyFallbackKernel) {
|
|
586
|
-
return null;
|
|
587
|
-
}
|
|
588
|
-
|
|
589
|
-
const result = cloneGraph(graph);
|
|
590
|
-
const keyMap = new Map();
|
|
591
|
-
const isF16Activation = ctx.activationDtype === 'f16';
|
|
592
|
-
|
|
593
|
-
// Build replacement kernel entries for each subgroup or fused-Q4K kernel
|
|
594
|
-
// reference found in decode, prefill, and postLayer steps.
|
|
595
|
-
const decodeKeys = new Set((result.decode || []).map((s) => s[1]));
|
|
596
|
-
const prefillKeys = new Set((result.prefill || []).map((s) => s[1]));
|
|
597
|
-
const postLayerKeys = new Set((result.postLayer || []).map((s) => s[1]));
|
|
598
|
-
const relevantKeys = new Set([...decodeKeys, ...prefillKeys, ...postLayerKeys]);
|
|
599
|
-
|
|
600
|
-
for (const key of relevantKeys) {
|
|
601
|
-
const entry = result.kernels[key];
|
|
602
|
-
if (!entry || !requiresNoSubgroupFallback(entry)) {
|
|
603
|
-
continue;
|
|
604
|
-
}
|
|
605
|
-
|
|
606
|
-
const isPostLayer = postLayerKeys.has(key) && !decodeKeys.has(key);
|
|
607
|
-
const isMulticol = entry.entry === 'main_multicol';
|
|
608
|
-
const isLmHead = isPostLayer || isMulticol;
|
|
609
|
-
|
|
610
|
-
let newFile;
|
|
611
|
-
let newEntry = 'main';
|
|
612
|
-
let newConstants = undefined;
|
|
613
|
-
|
|
614
|
-
if (entry.kernel === 'matmul_gemv_subgroup.wgsl') {
|
|
615
|
-
if (isLmHead) {
|
|
616
|
-
// lm_head: multicol → plain matmul, remove MULTICOL constants
|
|
617
|
-
newFile = 'matmul_f16w_f32a.wgsl';
|
|
618
|
-
newConstants = null;
|
|
619
|
-
} else {
|
|
620
|
-
// decode projections: vec4 → tiled matmul
|
|
621
|
-
newFile = 'matmul_f16w_f32a_tiled.wgsl';
|
|
622
|
-
}
|
|
623
|
-
} else if (entry.kernel === 'matmul_gemv_subgroup_f16a.wgsl') {
|
|
624
|
-
if (isLmHead) {
|
|
625
|
-
newFile = isF16Activation ? 'matmul_f16.wgsl' : 'matmul_f16w_f32a.wgsl';
|
|
626
|
-
newConstants = null;
|
|
627
|
-
} else {
|
|
628
|
-
newFile = isF16Activation ? 'matmul_f16.wgsl' : 'matmul_f16w_f32a_tiled.wgsl';
|
|
629
|
-
}
|
|
630
|
-
} else if (entry.kernel === 'attention_decode_online_f16kv.wgsl') {
|
|
631
|
-
// f16kv online uses f32 Q; if activations are f16, fall back to all-f16 chunked
|
|
632
|
-
newFile = isF16Activation
|
|
633
|
-
? 'attention_decode_chunked_f16.wgsl'
|
|
634
|
-
: 'attention_decode_chunked_f16kv.wgsl';
|
|
635
|
-
newEntry = entry.entry;
|
|
636
|
-
} else if (entry.kernel === 'attention_decode_online_f16.wgsl') {
|
|
637
|
-
newFile = 'attention_decode_chunked_f16.wgsl';
|
|
638
|
-
newEntry = entry.entry;
|
|
639
|
-
} else if (entry.kernel.startsWith('fused_matmul_q4')) {
|
|
640
|
-
newFile = isF16Activation ? 'matmul_f16_tiled.wgsl' : 'matmul_f16w_f32a_tiled.wgsl';
|
|
641
|
-
newConstants = null;
|
|
642
|
-
} else {
|
|
643
|
-
// Unknown subgroup kernel — skip
|
|
644
|
-
continue;
|
|
645
|
-
}
|
|
646
|
-
|
|
647
|
-
const newKey = deriveKernelKey(result.kernels, key, '_nosg');
|
|
648
|
-
result.kernels[newKey] = deriveKernelEntry(entry, newFile, newEntry, newConstants);
|
|
649
|
-
keyMap.set(key, newKey);
|
|
650
|
-
}
|
|
651
|
-
|
|
652
|
-
if (keyMap.size === 0) {
|
|
653
|
-
return null;
|
|
654
|
-
}
|
|
655
|
-
|
|
656
|
-
// Remap decode, prefill, and postLayer steps; leave preLayer untouched
|
|
657
|
-
result.decode = remapStepKeys(result.decode || [], keyMap);
|
|
658
|
-
result.prefill = remapStepKeys(result.prefill || [], keyMap);
|
|
659
|
-
result.postLayer = remapStepKeys(result.postLayer || [], keyMap);
|
|
660
|
-
|
|
661
|
-
return result;
|
|
662
|
-
}
|
|
663
154
|
|
|
664
155
|
// =============================================================================
|
|
665
156
|
// Transform: widenToF32Activations
|
|
@@ -670,25 +161,6 @@ export function removeSubgroups(graph, ctx) {
|
|
|
670
161
|
* that still use f16 for weights and KV cache. Requires shader-f16 for weight
|
|
671
162
|
* and KV buffer reads.
|
|
672
163
|
*/
|
|
673
|
-
const F16_TO_F32_ACTIVATION_MAP = new Map([
|
|
674
|
-
['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
|
|
675
|
-
['rope_f16.wgsl', 'rope.wgsl'],
|
|
676
|
-
['residual_f16.wgsl', 'residual.wgsl'],
|
|
677
|
-
['gelu_f16.wgsl', 'gelu.wgsl'],
|
|
678
|
-
['silu_f16.wgsl', 'silu.wgsl'],
|
|
679
|
-
['sample_f16.wgsl', 'sample.wgsl'],
|
|
680
|
-
['gather_f16.wgsl', 'gather.wgsl'],
|
|
681
|
-
['gather_f16_f16_out.wgsl', 'gather.wgsl'],
|
|
682
|
-
['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
|
|
683
|
-
['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
|
|
684
|
-
['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
|
|
685
|
-
['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
|
|
686
|
-
['attention_decode_online_f16.wgsl', 'attention_decode_online_f16kv.wgsl'],
|
|
687
|
-
['attention_decode_chunked_f16.wgsl', 'attention_decode_chunked_f16kv.wgsl'],
|
|
688
|
-
['attention_small_f16.wgsl', 'attention_small_f16kv.wgsl'],
|
|
689
|
-
['attention_streaming_f16.wgsl', 'attention_streaming_f16kv.wgsl'],
|
|
690
|
-
['attention_head512_f16.wgsl', 'attention_head512_f16kv.wgsl'],
|
|
691
|
-
]);
|
|
692
164
|
|
|
693
165
|
export function resolveF16ToF32ActivationKernel(kernel) {
|
|
694
166
|
return F16_TO_F32_ACTIVATION_MAP.get(kernel) ?? null;
|
|
@@ -702,25 +174,13 @@ export function resolveF16ToF32ActivationKernel(kernel) {
|
|
|
702
174
|
* runtime session explicitly requests f16 activations for an execution-v1
|
|
703
175
|
* graph that was authored with conservative f32 activation defaults.
|
|
704
176
|
*/
|
|
705
|
-
|
|
706
|
-
Array.from(F16_TO_F32_ACTIVATION_MAP.entries(), ([from, to]) => [to, from])
|
|
707
|
-
);
|
|
177
|
+
|
|
708
178
|
F32_TO_F16_ACTIVATION_MAP.set('gather.wgsl', 'gather_f16.wgsl');
|
|
709
179
|
F32_TO_F16_ACTIVATION_MAP.set('attention_head256_f16kv.wgsl', 'attention_small_f16.wgsl');
|
|
710
180
|
// head512 pure-f16 prefill is model-scoped below; keep generic Gemma 4 E2B
|
|
711
181
|
// f16 requests fail-closed until that path has its own evidence.
|
|
712
182
|
F32_TO_F16_ACTIVATION_MAP.delete('attention_head512_f16kv.wgsl');
|
|
713
183
|
|
|
714
|
-
function hasExplicitF32ActivationContract(entry) {
|
|
715
|
-
const precision = entry?.precision;
|
|
716
|
-
if (!precision || typeof precision !== 'object') {
|
|
717
|
-
return false;
|
|
718
|
-
}
|
|
719
|
-
return precision.activationDtype === 'f32'
|
|
720
|
-
|| precision.inputDtype === 'f32'
|
|
721
|
-
|| precision.outputDtype === 'f32';
|
|
722
|
-
}
|
|
723
|
-
|
|
724
184
|
/*
|
|
725
185
|
* Correctness fallback: preserve f16 weights where possible, but widen both
|
|
726
186
|
* activations and KV-cache interactions onto the stable f32 execution lane.
|
|
@@ -754,37 +214,6 @@ const F16_TO_F32_CORRECTNESS_FALLBACK_MAP = new Map([
|
|
|
754
214
|
* pure-f32 equivalent. Used when the GPU cannot compile any f16 WGSL at all.
|
|
755
215
|
* Covers f16-activation, f16-weight (f16w), and f16-KV (f16kv) kernels.
|
|
756
216
|
*/
|
|
757
|
-
const FULL_F32_SHADER_MAP = new Map([
|
|
758
|
-
// f16-activation utility kernels → f32
|
|
759
|
-
['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
|
|
760
|
-
['rope_f16.wgsl', 'rope.wgsl'],
|
|
761
|
-
['residual_f16.wgsl', 'residual.wgsl'],
|
|
762
|
-
['gelu_f16.wgsl', 'gelu.wgsl'],
|
|
763
|
-
['silu_f16.wgsl', 'silu.wgsl'],
|
|
764
|
-
['sample_f16.wgsl', 'sample.wgsl'],
|
|
765
|
-
['gather_f16.wgsl', 'gather.wgsl'],
|
|
766
|
-
['gather_f16_f16_out.wgsl', 'gather.wgsl'],
|
|
767
|
-
['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
|
|
768
|
-
// f16-activation matmul → f32
|
|
769
|
-
['matmul_gemv_subgroup_f16a.wgsl', 'matmul_f32.wgsl'],
|
|
770
|
-
['matmul_f16.wgsl', 'matmul_f32.wgsl'],
|
|
771
|
-
['matmul_f16_tiled.wgsl', 'matmul_f32.wgsl'],
|
|
772
|
-
// f16-weight + f32-activation matmul → f32
|
|
773
|
-
['matmul_gemv_subgroup.wgsl', 'matmul_f32.wgsl'],
|
|
774
|
-
['matmul_f16w_f32a.wgsl', 'matmul_f32.wgsl'],
|
|
775
|
-
['matmul_f16w_f32a_tiled.wgsl', 'matmul_f32.wgsl'],
|
|
776
|
-
// f16-activation attention → f32
|
|
777
|
-
['attention_decode_online_f16.wgsl', 'attention_streaming.wgsl'],
|
|
778
|
-
['attention_decode_chunked_f16.wgsl', 'attention_streaming.wgsl'],
|
|
779
|
-
['attention_small_f16.wgsl', 'attention_small.wgsl'],
|
|
780
|
-
['attention_streaming_f16.wgsl', 'attention_streaming.wgsl'],
|
|
781
|
-
// f16kv attention (f32 Q, f16 KV) → f32
|
|
782
|
-
['attention_decode_online_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
783
|
-
['attention_decode_chunked_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
784
|
-
['attention_small_f16kv.wgsl', 'attention_small.wgsl'],
|
|
785
|
-
['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
786
|
-
['attention_head256_f16kv.wgsl', 'attention_small.wgsl'],
|
|
787
|
-
]);
|
|
788
217
|
|
|
789
218
|
/*
|
|
790
219
|
* Widen all f16-activation shaders to f32-activation equivalents.
|
|
@@ -796,46 +225,6 @@ const FULL_F32_SHADER_MAP = new Map([
|
|
|
796
225
|
* to reflect the widened dtype.
|
|
797
226
|
*
|
|
798
227
|
*/
|
|
799
|
-
export function widenToF32Activations(graph, ctx) {
|
|
800
|
-
// Bail out if fused f16 FFN is present — no direct f32 equivalent
|
|
801
|
-
if (hasKernelFile(graph, 'fused_ffn_f16.wgsl')) {
|
|
802
|
-
return null;
|
|
803
|
-
}
|
|
804
|
-
|
|
805
|
-
// When the GPU cannot compile any f16 WGSL (hasF16=false), use the full f32
|
|
806
|
-
// map that also covers f16-weight and f16-KV kernels. Otherwise use the
|
|
807
|
-
// activation-only map that preserves f16 weights/KV for precision fallback.
|
|
808
|
-
const shaderMap = ctx.capabilities?.hasF16 === false || ctx.kvDtype === 'f32'
|
|
809
|
-
? FULL_F32_SHADER_MAP
|
|
810
|
-
: F16_TO_F32_ACTIVATION_MAP;
|
|
811
|
-
const fullF32 = ctx.capabilities?.hasF16 === false || ctx.kvDtype === 'f32';
|
|
812
|
-
|
|
813
|
-
const hasTargetShader = Object.values(graph.kernels).some(
|
|
814
|
-
(entry) => shaderMap.has(entry.kernel)
|
|
815
|
-
|| deriveQ4DecodeF32ActivationKernelEntry(entry)
|
|
816
|
-
|| deriveQ4PrefillF32ActivationKernelEntry(entry, { fullF32 })
|
|
817
|
-
);
|
|
818
|
-
if (!hasTargetShader) {
|
|
819
|
-
return null;
|
|
820
|
-
}
|
|
821
|
-
|
|
822
|
-
const result = cloneGraph(graph);
|
|
823
|
-
|
|
824
|
-
for (const [key, entry] of Object.entries(result.kernels)) {
|
|
825
|
-
const q4FallbackEntry = deriveQ4DecodeF32ActivationKernelEntry(entry)
|
|
826
|
-
?? deriveQ4PrefillF32ActivationKernelEntry(entry, { fullF32 });
|
|
827
|
-
if (q4FallbackEntry) {
|
|
828
|
-
result.kernels[key] = q4FallbackEntry;
|
|
829
|
-
continue;
|
|
830
|
-
}
|
|
831
|
-
const replacement = shaderMap.get(entry.kernel);
|
|
832
|
-
if (replacement !== undefined) {
|
|
833
|
-
result.kernels[key] = deriveMappedKernelEntry(entry, replacement);
|
|
834
|
-
}
|
|
835
|
-
}
|
|
836
|
-
|
|
837
|
-
return result;
|
|
838
|
-
}
|
|
839
228
|
|
|
840
229
|
/*
|
|
841
230
|
* Widen an f16 execution graph onto the stable f32 correctness lane used for
|
|
@@ -876,32 +265,8 @@ export function widenToF32CorrectnessFallback(graph, ctx) {
|
|
|
876
265
|
* Returns null if the graph has no supported f32-activation kernels to swap or
|
|
877
266
|
* if the runtime did not explicitly request f16 activations on an f16-capable
|
|
878
267
|
* GPU.
|
|
879
|
-
*
|
|
880
|
-
*/
|
|
881
|
-
export function narrowToF16Activations(graph, ctx) {
|
|
882
|
-
if (ctx.activationDtype !== 'f16' || ctx.capabilities?.hasF16 !== true) {
|
|
883
|
-
return null;
|
|
884
|
-
}
|
|
885
|
-
|
|
886
|
-
const hasTargetShader = Object.values(graph.kernels).some(
|
|
887
|
-
(entry) => !hasExplicitF32ActivationContract(entry) && F32_TO_F16_ACTIVATION_MAP.has(entry.kernel)
|
|
888
|
-
);
|
|
889
|
-
if (!hasTargetShader) {
|
|
890
|
-
return null;
|
|
891
|
-
}
|
|
892
|
-
|
|
893
|
-
const result = cloneGraph(graph);
|
|
894
|
-
for (const [key, entry] of Object.entries(result.kernels)) {
|
|
895
|
-
if (hasExplicitF32ActivationContract(entry)) {
|
|
896
|
-
continue;
|
|
897
|
-
}
|
|
898
|
-
const replacement = F32_TO_F16_ACTIVATION_MAP.get(entry.kernel);
|
|
899
|
-
if (replacement !== undefined) {
|
|
900
|
-
result.kernels[key] = deriveKernelEntry(entry, replacement, entry.entry);
|
|
901
|
-
}
|
|
902
|
-
}
|
|
903
|
-
return result;
|
|
904
|
-
}
|
|
268
|
+
*
|
|
269
|
+
*/
|
|
905
270
|
|
|
906
271
|
// =============================================================================
|
|
907
272
|
// Transform: swapPrefillAttention
|
|
@@ -1021,30 +386,10 @@ export function useHead256PrefillAttention(graph, ctx) {
|
|
|
1021
386
|
// Transform: widenProjectionWeightsToF32
|
|
1022
387
|
// =============================================================================
|
|
1023
388
|
|
|
1024
|
-
const PROJECTION_MATMUL_FILES = new Set([
|
|
1025
|
-
'matmul_gemv_subgroup.wgsl',
|
|
1026
|
-
'matmul_gemv_subgroup_f16a.wgsl',
|
|
1027
|
-
'matmul_f16w_f32a_tiled.wgsl',
|
|
1028
|
-
'matmul_f16w_f32a.wgsl',
|
|
1029
|
-
'matmul_f16.wgsl',
|
|
1030
|
-
'matmul_f16_tiled.wgsl',
|
|
1031
|
-
]);
|
|
1032
|
-
|
|
1033
389
|
/*
|
|
1034
390
|
* Known layer projection ops. Only these are widened; lm_head and embed are
|
|
1035
391
|
* excluded.
|
|
1036
392
|
*/
|
|
1037
|
-
const LAYER_PROJECTION_OPS = new Set([
|
|
1038
|
-
'q_proj', 'k_proj', 'v_proj', 'o_proj',
|
|
1039
|
-
'gate_proj', 'up_proj', 'down_proj',
|
|
1040
|
-
]);
|
|
1041
|
-
|
|
1042
|
-
const DENSE_Q4_PREFILL_FILES = new Set([
|
|
1043
|
-
'matmul_f16w_f32a.wgsl',
|
|
1044
|
-
'matmul_f16w_f32a_tiled.wgsl',
|
|
1045
|
-
'matmul_f16.wgsl',
|
|
1046
|
-
'matmul_f16_tiled.wgsl',
|
|
1047
|
-
]);
|
|
1048
393
|
|
|
1049
394
|
function resolveDensePrefillProjectionKernel(ctx) {
|
|
1050
395
|
return ctx.activationDtype === 'f16'
|
|
@@ -1061,51 +406,6 @@ function resolveDensePrefillProjectionKernel(ctx) {
|
|
|
1061
406
|
* Returns null if no applicable projection kernels are found.
|
|
1062
407
|
*
|
|
1063
408
|
*/
|
|
1064
|
-
export function widenProjectionWeightsToF32(graph, ctx) {
|
|
1065
|
-
// Collect kernel keys used by layer projection steps across all phases
|
|
1066
|
-
const projectionKernelKeys = new Set();
|
|
1067
|
-
const allPhases = ['preLayer', 'decode', 'prefill', 'postLayer'];
|
|
1068
|
-
|
|
1069
|
-
for (const phase of allPhases) {
|
|
1070
|
-
const steps = graph[phase];
|
|
1071
|
-
if (!Array.isArray(steps)) {
|
|
1072
|
-
continue;
|
|
1073
|
-
}
|
|
1074
|
-
for (const step of steps) {
|
|
1075
|
-
const op = step[0];
|
|
1076
|
-
const kernelKey = step[1];
|
|
1077
|
-
if (LAYER_PROJECTION_OPS.has(op) && kernelKey) {
|
|
1078
|
-
projectionKernelKeys.add(kernelKey);
|
|
1079
|
-
}
|
|
1080
|
-
}
|
|
1081
|
-
}
|
|
1082
|
-
|
|
1083
|
-
if (projectionKernelKeys.size === 0) {
|
|
1084
|
-
return null;
|
|
1085
|
-
}
|
|
1086
|
-
|
|
1087
|
-
// Check whether any of those keys reference a swappable matmul
|
|
1088
|
-
const keysToSwap = new Set();
|
|
1089
|
-
for (const key of projectionKernelKeys) {
|
|
1090
|
-
const entry = graph.kernels[key];
|
|
1091
|
-
if (entry && PROJECTION_MATMUL_FILES.has(entry.kernel)) {
|
|
1092
|
-
keysToSwap.add(key);
|
|
1093
|
-
}
|
|
1094
|
-
}
|
|
1095
|
-
|
|
1096
|
-
if (keysToSwap.size === 0) {
|
|
1097
|
-
return null;
|
|
1098
|
-
}
|
|
1099
|
-
|
|
1100
|
-
const result = cloneGraph(graph);
|
|
1101
|
-
|
|
1102
|
-
for (const key of keysToSwap) {
|
|
1103
|
-
const entry = result.kernels[key];
|
|
1104
|
-
result.kernels[key] = deriveKernelEntry(entry, 'matmul_f32.wgsl', 'main');
|
|
1105
|
-
}
|
|
1106
|
-
|
|
1107
|
-
return result;
|
|
1108
|
-
}
|
|
1109
409
|
|
|
1110
410
|
// =============================================================================
|
|
1111
411
|
// Transform: remapDenseQ4KPrefillToQ4Native
|
|
@@ -1123,68 +423,6 @@ export function widenProjectionWeightsToF32(graph, ctx) {
|
|
|
1123
423
|
* decode shape.
|
|
1124
424
|
*
|
|
1125
425
|
*/
|
|
1126
|
-
export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
|
|
1127
|
-
const densePrefillProjectionSteps = (graph.prefill || []).filter((step) => {
|
|
1128
|
-
if (!LAYER_PROJECTION_OPS.has(step[0])) {
|
|
1129
|
-
return false;
|
|
1130
|
-
}
|
|
1131
|
-
const entry = graph.kernels[step[1]];
|
|
1132
|
-
return entry != null && DENSE_Q4_PREFILL_FILES.has(entry.kernel);
|
|
1133
|
-
});
|
|
1134
|
-
if (densePrefillProjectionSteps.length === 0) {
|
|
1135
|
-
return null;
|
|
1136
|
-
}
|
|
1137
|
-
|
|
1138
|
-
const result = cloneGraph(graph);
|
|
1139
|
-
const existingSharedKey = findKernelKeyByFileAndEntry(
|
|
1140
|
-
result,
|
|
1141
|
-
'fused_matmul_q4_batched_multicol_shared.wgsl',
|
|
1142
|
-
'main'
|
|
1143
|
-
);
|
|
1144
|
-
let sharedKey = existingSharedKey;
|
|
1145
|
-
if (!sharedKey) {
|
|
1146
|
-
const q4DecodeKey = findPhaseKernelKey(
|
|
1147
|
-
graph,
|
|
1148
|
-
graph.decode || [],
|
|
1149
|
-
LAYER_PROJECTION_OPS,
|
|
1150
|
-
(entry) => entry.kernel === 'fused_matmul_q4.wgsl'
|
|
1151
|
-
);
|
|
1152
|
-
if (!q4DecodeKey) {
|
|
1153
|
-
return null;
|
|
1154
|
-
}
|
|
1155
|
-
const q4DecodeEntry = result.kernels[q4DecodeKey];
|
|
1156
|
-
sharedKey = deriveKernelKey(result.kernels, q4DecodeKey, '_prefill_shared');
|
|
1157
|
-
result.kernels[sharedKey] = deriveKernelEntry(
|
|
1158
|
-
q4DecodeEntry,
|
|
1159
|
-
'fused_matmul_q4_batched_multicol_shared.wgsl',
|
|
1160
|
-
'main',
|
|
1161
|
-
null
|
|
1162
|
-
);
|
|
1163
|
-
}
|
|
1164
|
-
|
|
1165
|
-
let changed = false;
|
|
1166
|
-
result.prefill = (result.prefill || []).map((step) => {
|
|
1167
|
-
const op = step[0];
|
|
1168
|
-
if (!LAYER_PROJECTION_OPS.has(op)) {
|
|
1169
|
-
return step;
|
|
1170
|
-
}
|
|
1171
|
-
const entry = result.kernels[step[1]];
|
|
1172
|
-
if (!entry || !DENSE_Q4_PREFILL_FILES.has(entry.kernel)) {
|
|
1173
|
-
return step;
|
|
1174
|
-
}
|
|
1175
|
-
|
|
1176
|
-
const replacementKey = sharedKey;
|
|
1177
|
-
if (replacementKey === step[1]) {
|
|
1178
|
-
return step;
|
|
1179
|
-
}
|
|
1180
|
-
changed = true;
|
|
1181
|
-
const next = [...step];
|
|
1182
|
-
next[1] = replacementKey;
|
|
1183
|
-
return next;
|
|
1184
|
-
});
|
|
1185
|
-
|
|
1186
|
-
return changed ? result : null;
|
|
1187
|
-
}
|
|
1188
426
|
|
|
1189
427
|
// =============================================================================
|
|
1190
428
|
// Transform: remapQ4KPrefillToDense
|
|
@@ -1595,74 +833,14 @@ export function remapQ4KDecodeFFNToGemv(graph, ctx) {
|
|
|
1595
833
|
* numerically aligned with the manifest-owned activation dtype.
|
|
1596
834
|
*
|
|
1597
835
|
*/
|
|
1598
|
-
export function useQwenDecodeF16Matmuls(graph, ctx) {
|
|
1599
|
-
const result = cloneGraph(graph);
|
|
1600
|
-
let changed = false;
|
|
1601
|
-
|
|
1602
|
-
for (const op of ['gate_proj', 'up_proj']) {
|
|
1603
|
-
const stepIndex = (result.decode || []).findIndex((entry) => Array.isArray(entry) && entry[0] === op);
|
|
1604
|
-
if (stepIndex === -1) {
|
|
1605
|
-
continue;
|
|
1606
|
-
}
|
|
1607
|
-
const step = result.decode[stepIndex];
|
|
1608
|
-
const kernelKey = step[1];
|
|
1609
|
-
const kernelEntry = result.kernels[kernelKey];
|
|
1610
|
-
if (!kernelEntry) {
|
|
1611
|
-
continue;
|
|
1612
|
-
}
|
|
1613
|
-
const derivedEntry = deriveLinearDecodeF16KernelEntry(kernelEntry);
|
|
1614
|
-
if (!derivedEntry) {
|
|
1615
|
-
continue;
|
|
1616
|
-
}
|
|
1617
|
-
const derivedKey = deriveKernelKey(result.kernels, kernelKey, '_decode_f16out');
|
|
1618
|
-
result.kernels[derivedKey] = derivedEntry;
|
|
1619
|
-
const replacement = [...step];
|
|
1620
|
-
replacement[1] = derivedKey;
|
|
1621
|
-
result.decode = [
|
|
1622
|
-
...result.decode.slice(0, stepIndex),
|
|
1623
|
-
replacement,
|
|
1624
|
-
...result.decode.slice(stepIndex + 1),
|
|
1625
|
-
];
|
|
1626
|
-
changed = true;
|
|
1627
|
-
}
|
|
1628
|
-
|
|
1629
|
-
const postLayerResult = replacePhaseStepKernelKey(
|
|
1630
|
-
result.postLayer ?? [],
|
|
1631
|
-
'lm_head',
|
|
1632
|
-
(() => {
|
|
1633
|
-
const lmHeadStep = (result.postLayer || []).find((entry) => Array.isArray(entry) && entry[0] === 'lm_head');
|
|
1634
|
-
if (!lmHeadStep) {
|
|
1635
|
-
return null;
|
|
1636
|
-
}
|
|
1637
|
-
const lmHeadKernelKey = lmHeadStep[1];
|
|
1638
|
-
const lmHeadKernel = result.kernels[lmHeadKernelKey];
|
|
1639
|
-
if (!lmHeadKernel) {
|
|
1640
|
-
return null;
|
|
1641
|
-
}
|
|
1642
|
-
const derivedEntry = deriveLmHeadDecodeF16KernelEntry(lmHeadKernel);
|
|
1643
|
-
if (!derivedEntry) {
|
|
1644
|
-
return null;
|
|
1645
|
-
}
|
|
1646
|
-
const derivedKey = deriveKernelKey(result.kernels, lmHeadKernelKey, '_decode_f16out');
|
|
1647
|
-
result.kernels[derivedKey] = derivedEntry;
|
|
1648
|
-
return derivedKey;
|
|
1649
|
-
})()
|
|
1650
|
-
);
|
|
1651
|
-
if (postLayerResult.changed) {
|
|
1652
|
-
result.postLayer = postLayerResult.steps;
|
|
1653
|
-
changed = true;
|
|
1654
|
-
}
|
|
1655
|
-
|
|
1656
|
-
return changed ? result : null;
|
|
1657
|
-
}
|
|
1658
836
|
|
|
1659
837
|
// =============================================================================
|
|
1660
838
|
// Transform: useQwenF16PrimaryMatmuls
|
|
1661
839
|
// =============================================================================
|
|
1662
840
|
|
|
1663
841
|
/*
|
|
1664
|
-
*
|
|
1665
|
-
*
|
|
842
|
+
* Select the Qwen 3.5 small-model execution graph's selective f16 probe lane
|
|
843
|
+
* when the runtime explicitly requests f16 activations.
|
|
1666
844
|
*
|
|
1667
845
|
* This transform narrows the decode projection and LM-head path while keeping
|
|
1668
846
|
* `o_proj` on the stable manifest-owned f32-output kernel. The prefill path is
|
|
@@ -1673,98 +851,6 @@ export function useQwenDecodeF16Matmuls(graph, ctx) {
|
|
|
1673
851
|
* post-attention RMSNorm).
|
|
1674
852
|
*
|
|
1675
853
|
*/
|
|
1676
|
-
export function useQwenF16PrimaryMatmuls(graph, ctx) {
|
|
1677
|
-
const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
|
|
1678
|
-
if (!layerTypes || layerTypes.length === 0) {
|
|
1679
|
-
return null;
|
|
1680
|
-
}
|
|
1681
|
-
|
|
1682
|
-
const result = cloneGraph(graph);
|
|
1683
|
-
let changed = false;
|
|
1684
|
-
|
|
1685
|
-
for (const [phaseName, op] of [['decode', 'attention'], ['prefill', 'attention']]) {
|
|
1686
|
-
const phaseSteps = result[phaseName] || [];
|
|
1687
|
-
const step = phaseSteps.find((entry) => Array.isArray(entry) && entry[0] === op);
|
|
1688
|
-
const kernelKey = step?.[1];
|
|
1689
|
-
const kernelEntry = kernelKey ? result.kernels[kernelKey] : null;
|
|
1690
|
-
const derivedEntry = deriveF16AttentionKernelEntry(kernelEntry);
|
|
1691
|
-
if (!step || !kernelKey || !derivedEntry) {
|
|
1692
|
-
continue;
|
|
1693
|
-
}
|
|
1694
|
-
const derivedKey = deriveKernelKey(result.kernels, kernelKey, '_primary_f16');
|
|
1695
|
-
result.kernels[derivedKey] = derivedEntry;
|
|
1696
|
-
const phaseResult = replacePhaseStepKernelKey(phaseSteps, op, derivedKey);
|
|
1697
|
-
if (phaseResult.changed) {
|
|
1698
|
-
result[phaseName] = phaseResult.steps;
|
|
1699
|
-
changed = true;
|
|
1700
|
-
}
|
|
1701
|
-
}
|
|
1702
|
-
|
|
1703
|
-
const decodeProjectionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
|
|
1704
|
-
const decodeProjectionKernel = decodeProjectionStep ? result.kernels[decodeProjectionStep[1]] : null;
|
|
1705
|
-
const decodeProjectionEntry = deriveQ4DecodeF16KernelEntry(decodeProjectionKernel);
|
|
1706
|
-
if (decodeProjectionStep && decodeProjectionEntry) {
|
|
1707
|
-
const decodeProjectionKey = deriveKernelKey(result.kernels, decodeProjectionStep[1], '_primary_f16');
|
|
1708
|
-
result.kernels[decodeProjectionKey] = decodeProjectionEntry;
|
|
1709
|
-
for (const op of ['q_proj', 'k_proj', 'v_proj', 'gate_proj', 'up_proj']) {
|
|
1710
|
-
const phaseResult = replacePhaseStepKernelKey(result.decode, op, decodeProjectionKey);
|
|
1711
|
-
if (phaseResult.changed) {
|
|
1712
|
-
result.decode = phaseResult.steps;
|
|
1713
|
-
changed = true;
|
|
1714
|
-
}
|
|
1715
|
-
}
|
|
1716
|
-
}
|
|
1717
|
-
|
|
1718
|
-
const prefillProjectionStep = (result.prefill || []).find((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
|
|
1719
|
-
const prefillProjectionKernel = prefillProjectionStep ? result.kernels[prefillProjectionStep[1]] : null;
|
|
1720
|
-
const prefillProjectionEntry = deriveQ4PrefillF16KernelEntry(prefillProjectionKernel);
|
|
1721
|
-
if (prefillProjectionStep && prefillProjectionEntry) {
|
|
1722
|
-
const prefillProjectionKey = deriveKernelKey(result.kernels, prefillProjectionStep[1], '_primary_f16');
|
|
1723
|
-
result.kernels[prefillProjectionKey] = prefillProjectionEntry;
|
|
1724
|
-
for (const op of ['q_proj', 'k_proj', 'v_proj']) {
|
|
1725
|
-
const phaseResult = replacePhaseStepKernelKey(result.prefill, op, prefillProjectionKey);
|
|
1726
|
-
if (phaseResult.changed) {
|
|
1727
|
-
result.prefill = phaseResult.steps;
|
|
1728
|
-
changed = true;
|
|
1729
|
-
}
|
|
1730
|
-
}
|
|
1731
|
-
}
|
|
1732
|
-
|
|
1733
|
-
for (const [phaseName, op] of [['decode', 'o_proj'], ['prefill', 'o_proj']]) {
|
|
1734
|
-
const phaseSteps = result[phaseName] || [];
|
|
1735
|
-
const step = phaseSteps.find((entry) => Array.isArray(entry) && entry[0] === op);
|
|
1736
|
-
const kernelKey = step?.[1];
|
|
1737
|
-
const kernelEntry = kernelKey ? result.kernels[kernelKey] : null;
|
|
1738
|
-
if (!step || !kernelKey || !kernelEntry) {
|
|
1739
|
-
continue;
|
|
1740
|
-
}
|
|
1741
|
-
const boundaryKey = deriveKernelKey(result.kernels, kernelKey, '_primary_f32_boundary');
|
|
1742
|
-
result.kernels[boundaryKey] = deriveKernelEntryWithPrecision(kernelEntry, {
|
|
1743
|
-
inputDtype: 'f32',
|
|
1744
|
-
outputDtype: 'f32',
|
|
1745
|
-
});
|
|
1746
|
-
const phaseResult = replacePhaseStepKernelKey(phaseSteps, op, boundaryKey);
|
|
1747
|
-
if (phaseResult.changed) {
|
|
1748
|
-
result[phaseName] = phaseResult.steps;
|
|
1749
|
-
changed = true;
|
|
1750
|
-
}
|
|
1751
|
-
}
|
|
1752
|
-
|
|
1753
|
-
const lmHeadStep = (result.postLayer || []).find((entry) => Array.isArray(entry) && entry[0] === 'lm_head');
|
|
1754
|
-
const lmHeadKernel = lmHeadStep ? result.kernels[lmHeadStep[1]] : null;
|
|
1755
|
-
const lmHeadEntry = deriveLmHeadDecodeF16KernelEntry(lmHeadKernel);
|
|
1756
|
-
if (lmHeadStep && lmHeadEntry) {
|
|
1757
|
-
const lmHeadKey = deriveKernelKey(result.kernels, lmHeadStep[1], '_primary_f16');
|
|
1758
|
-
result.kernels[lmHeadKey] = lmHeadEntry;
|
|
1759
|
-
const phaseResult = replacePhaseStepKernelKey(result.postLayer, 'lm_head', lmHeadKey);
|
|
1760
|
-
if (phaseResult.changed) {
|
|
1761
|
-
result.postLayer = phaseResult.steps;
|
|
1762
|
-
changed = true;
|
|
1763
|
-
}
|
|
1764
|
-
}
|
|
1765
|
-
|
|
1766
|
-
return changed ? result : null;
|
|
1767
|
-
}
|
|
1768
854
|
|
|
1769
855
|
// =============================================================================
|
|
1770
856
|
// Transform: useQwen36F16Activations
|
|
@@ -1774,93 +860,6 @@ export function useQwenF16PrimaryMatmuls(graph, ctx) {
|
|
|
1774
860
|
* Promote the Qwen 3.6 27B Q4K graph onto its additive all-f16 sibling lane.
|
|
1775
861
|
*
|
|
1776
862
|
*/
|
|
1777
|
-
export function useQwen36F16Activations(graph, ctx) {
|
|
1778
|
-
const narrowed = narrowToF16Activations(graph, ctx);
|
|
1779
|
-
const result = narrowed ?? cloneGraph(graph);
|
|
1780
|
-
let changed = narrowed != null;
|
|
1781
|
-
|
|
1782
|
-
const replaceKernelEntry = (key, entry) => {
|
|
1783
|
-
if (!key || !entry) {
|
|
1784
|
-
return;
|
|
1785
|
-
}
|
|
1786
|
-
result.kernels[key] = entry;
|
|
1787
|
-
changed = true;
|
|
1788
|
-
};
|
|
1789
|
-
const replaceOps = (phaseName, ops, kernelKey) => {
|
|
1790
|
-
for (const op of ops) {
|
|
1791
|
-
const phaseResult = replacePhaseStepKernelKey(result[phaseName], op, kernelKey);
|
|
1792
|
-
if (phaseResult.changed) {
|
|
1793
|
-
result[phaseName] = phaseResult.steps;
|
|
1794
|
-
changed = true;
|
|
1795
|
-
}
|
|
1796
|
-
}
|
|
1797
|
-
};
|
|
1798
|
-
|
|
1799
|
-
const embedStep = findPhaseStep(result.preLayer, 'embed');
|
|
1800
|
-
const embedKey = embedStep?.[1] ?? null;
|
|
1801
|
-
const embedEntry = embedKey ? result.kernels[embedKey] : null;
|
|
1802
|
-
if (embedEntry?.kernel === 'gather_f16.wgsl') {
|
|
1803
|
-
replaceKernelEntry(
|
|
1804
|
-
embedKey,
|
|
1805
|
-
deriveKernelEntryWithPrecision(
|
|
1806
|
-
deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
|
|
1807
|
-
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
1808
|
-
)
|
|
1809
|
-
);
|
|
1810
|
-
}
|
|
1811
|
-
|
|
1812
|
-
const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
|
|
1813
|
-
const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
|
|
1814
|
-
const decodeProjectionEntry = deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey]);
|
|
1815
|
-
if (decodeProjectionEntry) {
|
|
1816
|
-
const f16Key = deriveKernelKey(result.kernels, decodeProjectionKey, '_qwen36_f16');
|
|
1817
|
-
result.kernels[f16Key] = decodeProjectionEntry;
|
|
1818
|
-
replaceOps('decode', ['q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'], f16Key);
|
|
1819
|
-
}
|
|
1820
|
-
|
|
1821
|
-
const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
|
|
1822
|
-
const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
|
|
1823
|
-
const prefillProjectionEntry =
|
|
1824
|
-
deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
|
|
1825
|
-
?? deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
|
|
1826
|
-
?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey]);
|
|
1827
|
-
if (prefillProjectionEntry) {
|
|
1828
|
-
const f16Key = deriveKernelKey(result.kernels, prefillProjectionKey, '_qwen36_f16');
|
|
1829
|
-
result.kernels[f16Key] = prefillProjectionEntry;
|
|
1830
|
-
replaceOps('prefill', ['q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'], f16Key);
|
|
1831
|
-
}
|
|
1832
|
-
|
|
1833
|
-
const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
|
|
1834
|
-
const lmHeadKey = lmHeadStep?.[1] ?? null;
|
|
1835
|
-
replaceKernelEntry(
|
|
1836
|
-
lmHeadKey,
|
|
1837
|
-
deriveQ4DecodeF16KernelEntry(result.kernels[lmHeadKey])
|
|
1838
|
-
);
|
|
1839
|
-
|
|
1840
|
-
const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
|
|
1841
|
-
const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
|
|
1842
|
-
const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
|
|
1843
|
-
const lmHeadPrefillF16Entry =
|
|
1844
|
-
deriveQ4WideTilePrefillF16KernelEntry(lmHeadPrefillEntry)
|
|
1845
|
-
?? deriveQ4PrefillF16AccumKernelEntry(lmHeadPrefillEntry)
|
|
1846
|
-
?? deriveQ4PrefillF16KernelEntry(lmHeadPrefillEntry);
|
|
1847
|
-
if (lmHeadPrefillF16Entry) {
|
|
1848
|
-
replaceKernelEntry(lmHeadPrefillKey, lmHeadPrefillF16Entry);
|
|
1849
|
-
} else if (
|
|
1850
|
-
lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
|
|
1851
|
-
|| lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
|
|
1852
|
-
) {
|
|
1853
|
-
replaceKernelEntry(
|
|
1854
|
-
lmHeadPrefillKey,
|
|
1855
|
-
deriveKernelEntryWithPrecision(
|
|
1856
|
-
deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
|
|
1857
|
-
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
1858
|
-
)
|
|
1859
|
-
);
|
|
1860
|
-
}
|
|
1861
|
-
|
|
1862
|
-
return changed ? result : null;
|
|
1863
|
-
}
|
|
1864
863
|
|
|
1865
864
|
// =============================================================================
|
|
1866
865
|
// Transform: useGemma4Int4PleSelectiveF16Decode
|
|
@@ -1873,317 +872,11 @@ export function useQwen36F16Activations(graph, ctx) {
|
|
|
1873
872
|
* head512 prefill kernels.
|
|
1874
873
|
*
|
|
1875
874
|
*/
|
|
1876
|
-
export function useGemma4Int4PleSelectiveF16Decode(graph, ctx) {
|
|
1877
|
-
const result = cloneGraph(graph);
|
|
1878
|
-
let changed = false;
|
|
1879
|
-
|
|
1880
|
-
const decodeProjectionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
|
|
1881
|
-
const decodeProjectionKernel = decodeProjectionStep ? result.kernels[decodeProjectionStep[1]] : null;
|
|
1882
|
-
const decodeProjectionEntry = deriveDenseDecodeF16KernelEntry(decodeProjectionKernel);
|
|
1883
|
-
if (decodeProjectionStep && decodeProjectionEntry) {
|
|
1884
|
-
const decodeProjectionKey = deriveKernelKey(result.kernels, decodeProjectionStep[1], '_gemma4_f16');
|
|
1885
|
-
result.kernels[decodeProjectionKey] = decodeProjectionEntry;
|
|
1886
|
-
for (const op of ['q_proj', 'k_proj', 'v_proj']) {
|
|
1887
|
-
const phaseResult = replacePhaseStepKernelKey(result.decode, op, decodeProjectionKey);
|
|
1888
|
-
if (phaseResult.changed) {
|
|
1889
|
-
result.decode = phaseResult.steps;
|
|
1890
|
-
changed = true;
|
|
1891
|
-
}
|
|
1892
|
-
}
|
|
1893
|
-
}
|
|
1894
|
-
|
|
1895
|
-
for (const op of ['rope_q', 'rope_k']) {
|
|
1896
|
-
const step = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === op);
|
|
1897
|
-
const kernelKey = step?.[1];
|
|
1898
|
-
const kernelEntry = kernelKey ? result.kernels[kernelKey] : null;
|
|
1899
|
-
const replacement = kernelEntry ? F32_TO_F16_ACTIVATION_MAP.get(kernelEntry.kernel) : null;
|
|
1900
|
-
if (!step || !kernelKey || !replacement) {
|
|
1901
|
-
continue;
|
|
1902
|
-
}
|
|
1903
|
-
const ropeKey = deriveKernelKey(result.kernels, kernelKey, '_gemma4_f16');
|
|
1904
|
-
result.kernels[ropeKey] = deriveKernelEntryWithPrecision(
|
|
1905
|
-
deriveKernelEntry(kernelEntry, replacement, kernelEntry.entry),
|
|
1906
|
-
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
1907
|
-
);
|
|
1908
|
-
const phaseResult = replacePhaseStepKernelKey(result.decode, op, ropeKey);
|
|
1909
|
-
if (phaseResult.changed) {
|
|
1910
|
-
result.decode = phaseResult.steps;
|
|
1911
|
-
changed = true;
|
|
1912
|
-
}
|
|
1913
|
-
}
|
|
1914
|
-
|
|
1915
|
-
const attentionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'attention');
|
|
1916
|
-
const attentionKernel = attentionStep ? result.kernels[attentionStep[1]] : null;
|
|
1917
|
-
const attentionEntry = deriveF16AttentionKernelEntry(attentionKernel);
|
|
1918
|
-
if (attentionStep && attentionEntry) {
|
|
1919
|
-
const attentionKey = deriveKernelKey(result.kernels, attentionStep[1], '_gemma4_f16');
|
|
1920
|
-
result.kernels[attentionKey] = attentionEntry;
|
|
1921
|
-
const phaseResult = replacePhaseStepKernelKey(result.decode, 'attention', attentionKey);
|
|
1922
|
-
if (phaseResult.changed) {
|
|
1923
|
-
result.decode = phaseResult.steps;
|
|
1924
|
-
changed = true;
|
|
1925
|
-
}
|
|
1926
|
-
}
|
|
1927
|
-
|
|
1928
|
-
const oProjStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'o_proj');
|
|
1929
|
-
const oProjKernel = oProjStep ? result.kernels[oProjStep[1]] : null;
|
|
1930
|
-
if (oProjStep && oProjKernel) {
|
|
1931
|
-
const oProjKey = deriveKernelKey(result.kernels, oProjStep[1], '_gemma4_f32_boundary');
|
|
1932
|
-
result.kernels[oProjKey] = deriveKernelEntryWithPrecision(oProjKernel, {
|
|
1933
|
-
inputDtype: 'f32',
|
|
1934
|
-
outputDtype: 'f32',
|
|
1935
|
-
});
|
|
1936
|
-
const phaseResult = replacePhaseStepKernelKey(result.decode, 'o_proj', oProjKey);
|
|
1937
|
-
if (phaseResult.changed) {
|
|
1938
|
-
result.decode = phaseResult.steps;
|
|
1939
|
-
changed = true;
|
|
1940
|
-
}
|
|
1941
|
-
}
|
|
1942
|
-
|
|
1943
|
-
return changed ? result : null;
|
|
1944
|
-
}
|
|
1945
875
|
|
|
1946
876
|
// =============================================================================
|
|
1947
877
|
// Transform: useGemma4TextF16Activations
|
|
1948
878
|
// =============================================================================
|
|
1949
879
|
|
|
1950
|
-
const GEMMA4_12B_PREFILL_F32_PROJECTION_OPS = new Set([
|
|
1951
|
-
'q_proj',
|
|
1952
|
-
'k_proj',
|
|
1953
|
-
'v_proj',
|
|
1954
|
-
'gate_proj',
|
|
1955
|
-
'up_proj',
|
|
1956
|
-
]);
|
|
1957
|
-
|
|
1958
|
-
function remapGemma412BPrefillProjectionEntries(entries, f16Key, f32Key) {
|
|
1959
|
-
if (!Array.isArray(entries) || !f16Key || !f32Key) {
|
|
1960
|
-
return { entries, changed: false };
|
|
1961
|
-
}
|
|
1962
|
-
|
|
1963
|
-
let changed = false;
|
|
1964
|
-
const remapStep = (step) => {
|
|
1965
|
-
if (!Array.isArray(step) || !LAYER_PROJECTION_OPS.has(step[0])) {
|
|
1966
|
-
return step;
|
|
1967
|
-
}
|
|
1968
|
-
const targetKey = GEMMA4_12B_PREFILL_F32_PROJECTION_OPS.has(step[0])
|
|
1969
|
-
? f32Key
|
|
1970
|
-
: f16Key;
|
|
1971
|
-
if (step[1] === targetKey) {
|
|
1972
|
-
return step;
|
|
1973
|
-
}
|
|
1974
|
-
const replacement = [...step];
|
|
1975
|
-
replacement[1] = targetKey;
|
|
1976
|
-
changed = true;
|
|
1977
|
-
return replacement;
|
|
1978
|
-
};
|
|
1979
|
-
|
|
1980
|
-
const nextEntries = entries.map((entry) => {
|
|
1981
|
-
if (Array.isArray(entry)) {
|
|
1982
|
-
return remapStep(entry);
|
|
1983
|
-
}
|
|
1984
|
-
if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
|
|
1985
|
-
return entry;
|
|
1986
|
-
}
|
|
1987
|
-
return {
|
|
1988
|
-
...entry,
|
|
1989
|
-
steps: entry.steps.map((step) => remapStep(step)),
|
|
1990
|
-
};
|
|
1991
|
-
});
|
|
1992
|
-
|
|
1993
|
-
return { entries: nextEntries, changed };
|
|
1994
|
-
}
|
|
1995
|
-
|
|
1996
|
-
function remapGemma412BStableBoundaryEntries(result, sourceGraph) {
|
|
1997
|
-
let changed = false;
|
|
1998
|
-
|
|
1999
|
-
const replaceStepWithSourceEntry = (phaseName, op, precision) => {
|
|
2000
|
-
const phase = result[phaseName];
|
|
2001
|
-
const sourcePhase = sourceGraph[phaseName];
|
|
2002
|
-
const step = findPhaseStep(phase, op);
|
|
2003
|
-
const sourceStep = findPhaseStep(sourcePhase, op);
|
|
2004
|
-
const sourceKey = sourceStep?.[1] ?? step?.[1] ?? null;
|
|
2005
|
-
const sourceEntry = sourceKey ? sourceGraph.kernels[sourceKey] : null;
|
|
2006
|
-
if (!step || !sourceKey || !sourceEntry) {
|
|
2007
|
-
return;
|
|
2008
|
-
}
|
|
2009
|
-
const stableKey = deriveKernelKey(result.kernels, sourceKey, '_gemma4_12b_stable');
|
|
2010
|
-
result.kernels[stableKey] = precision
|
|
2011
|
-
? deriveKernelEntryWithPrecision(sourceEntry, precision)
|
|
2012
|
-
: { ...sourceEntry };
|
|
2013
|
-
const phaseResult = replacePhaseStepKernelKey(phase, op, stableKey);
|
|
2014
|
-
if (phaseResult.changed) {
|
|
2015
|
-
result[phaseName] = phaseResult.steps;
|
|
2016
|
-
changed = true;
|
|
2017
|
-
}
|
|
2018
|
-
};
|
|
2019
|
-
|
|
2020
|
-
for (const op of ['q_proj', 'k_proj', 'v_proj', 'rope_q', 'rope_k']) {
|
|
2021
|
-
replaceStepWithSourceEntry('decode', op, { inputDtype: 'f32', outputDtype: 'f32' });
|
|
2022
|
-
}
|
|
2023
|
-
replaceStepWithSourceEntry('decode', 'attention', {
|
|
2024
|
-
activationDtype: 'f32',
|
|
2025
|
-
kvDtype: 'f16',
|
|
2026
|
-
outputDtype: 'f32',
|
|
2027
|
-
});
|
|
2028
|
-
for (const op of ['final_norm', 'lm_head', 'lm_head_prefill', 'sample']) {
|
|
2029
|
-
replaceStepWithSourceEntry('postLayer', op, null);
|
|
2030
|
-
}
|
|
2031
|
-
|
|
2032
|
-
return changed;
|
|
2033
|
-
}
|
|
2034
|
-
|
|
2035
|
-
function useGemma4TextF16ActivationsForLane(graph, ctx, options) {
|
|
2036
|
-
const narrowed = narrowToF16Activations(graph, ctx);
|
|
2037
|
-
const result = narrowed ?? cloneGraph(graph);
|
|
2038
|
-
let changed = narrowed != null;
|
|
2039
|
-
|
|
2040
|
-
const replaceKernelEntry = (key, entry) => {
|
|
2041
|
-
if (!key || !entry) {
|
|
2042
|
-
return;
|
|
2043
|
-
}
|
|
2044
|
-
result.kernels[key] = entry;
|
|
2045
|
-
changed = true;
|
|
2046
|
-
};
|
|
2047
|
-
const stableTextBoundary = options?.stableTextBoundary === true;
|
|
2048
|
-
|
|
2049
|
-
const derivePrefillAttentionEntry = (entry) => {
|
|
2050
|
-
if (
|
|
2051
|
-
stableTextBoundary
|
|
2052
|
-
&& typeof entry?.kernel === 'string'
|
|
2053
|
-
&& entry.kernel === 'attention_small_f16.wgsl'
|
|
2054
|
-
) {
|
|
2055
|
-
return deriveKernelEntryWithPrecision(
|
|
2056
|
-
deriveKernelEntry(entry, 'attention_head256_f16kv.wgsl', 'main'),
|
|
2057
|
-
{ activationDtype: 'f32', kvDtype: 'f16', outputDtype: 'f32' }
|
|
2058
|
-
);
|
|
2059
|
-
}
|
|
2060
|
-
if (
|
|
2061
|
-
stableTextBoundary
|
|
2062
|
-
&& typeof entry?.kernel === 'string'
|
|
2063
|
-
&& entry.kernel.endsWith('_f16kv.wgsl')
|
|
2064
|
-
) {
|
|
2065
|
-
return deriveKernelEntryWithPrecision(
|
|
2066
|
-
entry,
|
|
2067
|
-
{ activationDtype: 'f32', kvDtype: 'f16', outputDtype: 'f32' }
|
|
2068
|
-
);
|
|
2069
|
-
}
|
|
2070
|
-
return deriveF16AttentionKernelEntry(entry);
|
|
2071
|
-
};
|
|
2072
|
-
|
|
2073
|
-
const embedStep = findPhaseStep(result.preLayer, 'embed');
|
|
2074
|
-
const embedKey = embedStep?.[1] ?? null;
|
|
2075
|
-
const embedEntry = embedKey ? result.kernels[embedKey] : null;
|
|
2076
|
-
if (embedEntry?.kernel === 'gather_f16.wgsl') {
|
|
2077
|
-
replaceKernelEntry(
|
|
2078
|
-
embedKey,
|
|
2079
|
-
deriveKernelEntryWithPrecision(
|
|
2080
|
-
deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
|
|
2081
|
-
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2082
|
-
)
|
|
2083
|
-
);
|
|
2084
|
-
}
|
|
2085
|
-
|
|
2086
|
-
const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
|
|
2087
|
-
const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
|
|
2088
|
-
replaceKernelEntry(
|
|
2089
|
-
decodeProjectionKey,
|
|
2090
|
-
deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey])
|
|
2091
|
-
);
|
|
2092
|
-
|
|
2093
|
-
const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
|
|
2094
|
-
const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
|
|
2095
|
-
const sourcePrefillProjectionEntry = result.kernels[prefillProjectionKey];
|
|
2096
|
-
const prefillProjectionF16Entry = stableTextBoundary
|
|
2097
|
-
? (
|
|
2098
|
-
deriveQ4WideTilePrefillF16KernelEntry(sourcePrefillProjectionEntry)
|
|
2099
|
-
?? deriveQ4PrefillF16AccumKernelEntry(sourcePrefillProjectionEntry)
|
|
2100
|
-
?? deriveQ4PrefillF16KernelEntry(sourcePrefillProjectionEntry)
|
|
2101
|
-
)
|
|
2102
|
-
: (
|
|
2103
|
-
deriveQ4PrefillF16AccumKernelEntry(sourcePrefillProjectionEntry)
|
|
2104
|
-
?? deriveQ4WideTilePrefillF16KernelEntry(sourcePrefillProjectionEntry)
|
|
2105
|
-
?? deriveQ4PrefillF16KernelEntry(sourcePrefillProjectionEntry)
|
|
2106
|
-
);
|
|
2107
|
-
if (stableTextBoundary && prefillProjectionKey && prefillProjectionF16Entry) {
|
|
2108
|
-
const prefillProjectionF16Key = deriveKernelKey(result.kernels, prefillProjectionKey, '_gemma4_f16');
|
|
2109
|
-
result.kernels[prefillProjectionF16Key] = prefillProjectionF16Entry;
|
|
2110
|
-
changed = true;
|
|
2111
|
-
const remapped = remapGemma412BPrefillProjectionEntries(
|
|
2112
|
-
result.prefill,
|
|
2113
|
-
prefillProjectionF16Key,
|
|
2114
|
-
prefillProjectionKey
|
|
2115
|
-
);
|
|
2116
|
-
result.prefill = remapped.entries;
|
|
2117
|
-
changed = changed || remapped.changed;
|
|
2118
|
-
} else {
|
|
2119
|
-
replaceKernelEntry(
|
|
2120
|
-
prefillProjectionKey,
|
|
2121
|
-
prefillProjectionF16Entry
|
|
2122
|
-
);
|
|
2123
|
-
}
|
|
2124
|
-
|
|
2125
|
-
const replacePrefillAttentionEntries = (entries) => {
|
|
2126
|
-
for (const entry of entries || []) {
|
|
2127
|
-
if (Array.isArray(entry)) {
|
|
2128
|
-
if (entry[0] !== 'attention') {
|
|
2129
|
-
continue;
|
|
2130
|
-
}
|
|
2131
|
-
replaceKernelEntry(
|
|
2132
|
-
entry[1],
|
|
2133
|
-
derivePrefillAttentionEntry(result.kernels[entry[1]])
|
|
2134
|
-
);
|
|
2135
|
-
continue;
|
|
2136
|
-
}
|
|
2137
|
-
if (entry && typeof entry === 'object' && Array.isArray(entry.steps)) {
|
|
2138
|
-
replacePrefillAttentionEntries(entry.steps);
|
|
2139
|
-
}
|
|
2140
|
-
}
|
|
2141
|
-
};
|
|
2142
|
-
replacePrefillAttentionEntries(result.prefill);
|
|
2143
|
-
|
|
2144
|
-
const finalNormStep = findPhaseStep(result.postLayer, 'final_norm');
|
|
2145
|
-
const finalNormKey = finalNormStep?.[1] ?? null;
|
|
2146
|
-
const finalNormEntry = finalNormKey ? result.kernels[finalNormKey] : null;
|
|
2147
|
-
if (finalNormEntry?.kernel === 'rmsnorm.wgsl') {
|
|
2148
|
-
replaceKernelEntry(
|
|
2149
|
-
finalNormKey,
|
|
2150
|
-
deriveKernelEntryWithPrecision(
|
|
2151
|
-
deriveKernelEntry(finalNormEntry, 'rmsnorm_f16.wgsl', finalNormEntry.entry),
|
|
2152
|
-
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2153
|
-
)
|
|
2154
|
-
);
|
|
2155
|
-
}
|
|
2156
|
-
|
|
2157
|
-
const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
|
|
2158
|
-
const lmHeadKey = lmHeadStep?.[1] ?? null;
|
|
2159
|
-
replaceKernelEntry(
|
|
2160
|
-
lmHeadKey,
|
|
2161
|
-
deriveLmHeadDecodeF16KernelEntry(result.kernels[lmHeadKey])
|
|
2162
|
-
);
|
|
2163
|
-
|
|
2164
|
-
const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
|
|
2165
|
-
const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
|
|
2166
|
-
const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
|
|
2167
|
-
if (
|
|
2168
|
-
lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
|
|
2169
|
-
|| lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
|
|
2170
|
-
) {
|
|
2171
|
-
replaceKernelEntry(
|
|
2172
|
-
lmHeadPrefillKey,
|
|
2173
|
-
deriveKernelEntryWithPrecision(
|
|
2174
|
-
deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
|
|
2175
|
-
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2176
|
-
)
|
|
2177
|
-
);
|
|
2178
|
-
}
|
|
2179
|
-
|
|
2180
|
-
if (stableTextBoundary) {
|
|
2181
|
-
changed = remapGemma412BStableBoundaryEntries(result, graph) || changed;
|
|
2182
|
-
}
|
|
2183
|
-
|
|
2184
|
-
return changed ? result : null;
|
|
2185
|
-
}
|
|
2186
|
-
|
|
2187
880
|
export function useGemma4TextF16Activations(graph, ctx) {
|
|
2188
881
|
return useGemma4TextF16ActivationsForLane(graph, ctx, { stableTextBoundary: false });
|
|
2189
882
|
}
|
|
@@ -2203,112 +896,13 @@ export function useGemma431BTextF16Activations(graph, ctx) {
|
|
|
2203
896
|
/*
|
|
2204
897
|
* Promote the Gemma 4 E2B INT4-PLE Q4K graph onto the all-f16 lane via the
|
|
2205
898
|
* weights-ref sibling manifest gemma-4-e2b-it-q4k-ehf16-af16-int4ple. Mirrors
|
|
2206
|
-
* useGemma4TextF16Activations: same Q4 weight
|
|
899
|
+
* useGemma4TextF16Activations: same Q4 weight capsule, kernels narrowed to f16
|
|
2207
900
|
* activations, prefill projections promoted from widetile to widetile_f16a,
|
|
2208
901
|
* decode projections to multicol_f16a, lm_head/sample/final_norm to their f16
|
|
2209
902
|
* counterparts. Apple Metal stays disabled at the capability layer because the
|
|
2210
903
|
* fused-q4k+f16 kernel pool produces NaN at L0.ffn_down on metal-3.
|
|
2211
904
|
*
|
|
2212
905
|
*/
|
|
2213
|
-
export function useGemma4Int4PleAf16Activations(graph, ctx) {
|
|
2214
|
-
const narrowed = narrowToF16Activations(graph, ctx);
|
|
2215
|
-
const result = narrowed ?? cloneGraph(graph);
|
|
2216
|
-
let changed = narrowed != null;
|
|
2217
|
-
|
|
2218
|
-
const replaceKernelEntry = (key, entry) => {
|
|
2219
|
-
if (!key || !entry) {
|
|
2220
|
-
return;
|
|
2221
|
-
}
|
|
2222
|
-
result.kernels[key] = entry;
|
|
2223
|
-
changed = true;
|
|
2224
|
-
};
|
|
2225
|
-
|
|
2226
|
-
const embedStep = findPhaseStep(result.preLayer, 'embed');
|
|
2227
|
-
const embedKey = embedStep?.[1] ?? null;
|
|
2228
|
-
const embedEntry = embedKey ? result.kernels[embedKey] : null;
|
|
2229
|
-
if (embedEntry?.kernel === 'gather_f16.wgsl') {
|
|
2230
|
-
replaceKernelEntry(
|
|
2231
|
-
embedKey,
|
|
2232
|
-
deriveKernelEntryWithPrecision(
|
|
2233
|
-
deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
|
|
2234
|
-
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2235
|
-
)
|
|
2236
|
-
);
|
|
2237
|
-
}
|
|
2238
|
-
|
|
2239
|
-
const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
|
|
2240
|
-
const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
|
|
2241
|
-
replaceKernelEntry(
|
|
2242
|
-
decodeProjectionKey,
|
|
2243
|
-
deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey])
|
|
2244
|
-
);
|
|
2245
|
-
|
|
2246
|
-
const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
|
|
2247
|
-
const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
|
|
2248
|
-
replaceKernelEntry(
|
|
2249
|
-
prefillProjectionKey,
|
|
2250
|
-
deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
|
|
2251
|
-
?? deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
|
|
2252
|
-
?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey])
|
|
2253
|
-
);
|
|
2254
|
-
|
|
2255
|
-
const replacePrefillAttentionEntries = (entries) => {
|
|
2256
|
-
for (const entry of entries || []) {
|
|
2257
|
-
if (Array.isArray(entry)) {
|
|
2258
|
-
if (entry[0] !== 'attention') {
|
|
2259
|
-
continue;
|
|
2260
|
-
}
|
|
2261
|
-
replaceKernelEntry(
|
|
2262
|
-
entry[1],
|
|
2263
|
-
deriveF16AttentionKernelEntry(result.kernels[entry[1]])
|
|
2264
|
-
);
|
|
2265
|
-
continue;
|
|
2266
|
-
}
|
|
2267
|
-
if (entry && typeof entry === 'object' && Array.isArray(entry.steps)) {
|
|
2268
|
-
replacePrefillAttentionEntries(entry.steps);
|
|
2269
|
-
}
|
|
2270
|
-
}
|
|
2271
|
-
};
|
|
2272
|
-
replacePrefillAttentionEntries(result.prefill);
|
|
2273
|
-
|
|
2274
|
-
const finalNormStep = findPhaseStep(result.postLayer, 'final_norm');
|
|
2275
|
-
const finalNormKey = finalNormStep?.[1] ?? null;
|
|
2276
|
-
const finalNormEntry = finalNormKey ? result.kernels[finalNormKey] : null;
|
|
2277
|
-
if (finalNormEntry?.kernel === 'rmsnorm.wgsl') {
|
|
2278
|
-
replaceKernelEntry(
|
|
2279
|
-
finalNormKey,
|
|
2280
|
-
deriveKernelEntryWithPrecision(
|
|
2281
|
-
deriveKernelEntry(finalNormEntry, 'rmsnorm_f16.wgsl', finalNormEntry.entry),
|
|
2282
|
-
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2283
|
-
)
|
|
2284
|
-
);
|
|
2285
|
-
}
|
|
2286
|
-
|
|
2287
|
-
const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
|
|
2288
|
-
const lmHeadKey = lmHeadStep?.[1] ?? null;
|
|
2289
|
-
replaceKernelEntry(
|
|
2290
|
-
lmHeadKey,
|
|
2291
|
-
deriveLmHeadDecodeF16KernelEntry(result.kernels[lmHeadKey])
|
|
2292
|
-
);
|
|
2293
|
-
|
|
2294
|
-
const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
|
|
2295
|
-
const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
|
|
2296
|
-
const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
|
|
2297
|
-
if (
|
|
2298
|
-
lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
|
|
2299
|
-
|| lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
|
|
2300
|
-
) {
|
|
2301
|
-
replaceKernelEntry(
|
|
2302
|
-
lmHeadPrefillKey,
|
|
2303
|
-
deriveKernelEntryWithPrecision(
|
|
2304
|
-
deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
|
|
2305
|
-
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2306
|
-
)
|
|
2307
|
-
);
|
|
2308
|
-
}
|
|
2309
|
-
|
|
2310
|
-
return changed ? result : null;
|
|
2311
|
-
}
|
|
2312
906
|
|
|
2313
907
|
// =============================================================================
|
|
2314
908
|
// Composition
|