doppler-gpu 0.4.7 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +93 -144
- package/models/catalog.json +1575 -0
- package/package.json +37 -12
- package/src/cli/doppler-cli.js +28 -7
- package/src/client/doppler-registry.json +2 -2
- package/src/client/runtime/index.js +25 -2
- package/src/client/runtime/model-session.d.ts +31 -1
- package/src/client/runtime/model-session.js +31 -0
- package/src/client/runtime/model-source.d.ts +19 -2
- package/src/client/runtime/model-source.js +45 -5
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
- package/src/config/kernel-path-loader.js +1 -0
- package/src/config/kernels/kernel-ref-digests.js +26 -17
- package/src/config/kernels/registry.json +448 -27
- package/src/config/merge.js +11 -0
- package/src/config/param-validator.js +25 -0
- package/src/config/runtime/default.json +3 -1
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
- package/src/config/runtime/profiles/rerank-stability.json +43 -0
- package/src/config/runtime/profiles/rerank-throughput.json +40 -0
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
- package/src/config/schema/doppler.schema.js +40 -5
- package/src/config/schema/execution-v1.schema.d.ts +41 -0
- package/src/config/schema/execution-v1.schema.js +10 -0
- package/src/config/schema/index.d.ts +3 -1
- package/src/config/schema/speculation-self.schema.d.ts +16 -0
- package/src/config/transforms/capability-transform-resolver.js +7 -1
- package/src/config/transforms/execution-graph-transforms.js +2 -2
- package/src/debug/config.d.ts +4 -0
- package/src/experimental/orchestration/index.d.ts +10 -1
- package/src/experimental/training/suite.js +1 -1
- package/src/formats/rdrr/classification.js +3 -0
- package/src/formats/rdrr/integrity.d.ts +0 -2
- package/src/formats/rdrr/integrity.js +0 -2
- package/src/formats/rdrr/types.d.ts +1 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +75 -16
- package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
- package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
- package/src/gpu/kernels/fused_ffn.d.ts +32 -0
- package/src/gpu/kernels/fused_ffn.js +540 -19
- package/src/gpu/kernels/fused_ffn.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
- package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
- package/src/gpu/kernels/index.d.ts +21 -0
- package/src/gpu/kernels/index.js +13 -0
- package/src/gpu/kernels/linear-attention-core.js +413 -15
- package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
- package/src/gpu/kernels/lm-head-argmax.js +153 -57
- package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
- package/src/gpu/kernels/lm-head-select-logits.js +186 -0
- package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
- package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
- package/src/gpu/kernels/matmul-selection.js +5 -1
- package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
- package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
- package/src/gpu/kernels/rmsnorm-pair.js +113 -0
- package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
- package/src/gpu/kernels/rmsnorm-stats.js +250 -0
- package/src/gpu/kernels/silu.wgsl +7 -1
- package/src/gpu/kernels/silu_f16.wgsl +7 -1
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
- package/src/gpu/tensor.d.ts +1 -0
- package/src/index.d.ts +1 -1
- package/src/index.js +1 -1
- package/src/inference/browser-harness-suite-helpers.js +1 -1
- package/src/inference/browser-harness-text-helpers.js +251 -20
- package/src/inference/browser-harness.js +221 -3
- package/src/inference/kv-cache/base.js +42 -1
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
- package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
- package/src/inference/pipelines/text/attention/projections.js +174 -4
- package/src/inference/pipelines/text/attention/record.js +44 -21
- package/src/inference/pipelines/text/attention/run.js +18 -3
- package/src/inference/pipelines/text/config.d.ts +18 -0
- package/src/inference/pipelines/text/config.js +100 -0
- package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
- package/src/inference/pipelines/text/execution-plan.js +37 -0
- package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
- package/src/inference/pipelines/text/execution-v1.js +122 -8
- package/src/inference/pipelines/text/ffn/dense.js +239 -21
- package/src/inference/pipelines/text/ffn/standard.js +121 -38
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
- package/src/inference/pipelines/text/generator-helpers.js +2 -0
- package/src/inference/pipelines/text/generator-runtime.js +1 -0
- package/src/inference/pipelines/text/generator-steps.js +2 -2
- package/src/inference/pipelines/text/generator.d.ts +18 -1
- package/src/inference/pipelines/text/generator.js +297 -19
- package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
- package/src/inference/pipelines/text/init.d.ts +22 -3
- package/src/inference/pipelines/text/init.js +2 -2
- package/src/inference/pipelines/text/layer.js +87 -8
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
- package/src/inference/pipelines/text/linear-attention.js +173 -16
- package/src/inference/pipelines/text/logits/gpu.js +2 -2
- package/src/inference/pipelines/text/logits/index.js +62 -1
- package/src/inference/pipelines/text/model-load.js +6 -0
- package/src/inference/pipelines/text/ops.js +22 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
- package/src/inference/pipelines/text/probes.js +9 -4
- package/src/inference/pipelines/text/state.d.ts +4 -4
- package/src/inference/pipelines/text/state.js +4 -0
- package/src/inference/pipelines/text/types.d.ts +27 -2
- package/src/inference/pipelines/text.d.ts +21 -1
- package/src/inference/pipelines/text.js +40 -14
- package/src/inference/tokenizers/bundled.js +45 -23
- package/src/inference/tokenizers/types.d.ts +7 -4
- package/src/loader/doppler-loader.js +25 -2
- package/src/loader/index.d.ts +3 -1
- package/src/loader/index.js +2 -0
- package/src/loader/loader-types.d.ts +3 -1
- package/src/loader/manifest-config.d.ts +18 -0
- package/src/loader/manifest-config.js +11 -3
- package/src/loader/shard-cache.d.ts +2 -0
- package/src/loader/shard-cache.js +29 -1
- package/src/models/gemma4.js +8 -0
- package/src/rules/inference/capability-transforms.rules.json +145 -20
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/fused-ffn.rules.json +1 -1
- package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/rules/kernels/silu.rules.json +1 -0
- package/src/rules/rule-registry.js +2 -0
- package/src/storage/artifact-storage-context.js +36 -0
- package/src/tooling/node-browser-command-runner.js +42 -7
- package/src/tooling/opfs-cache.d.ts +21 -1
- package/src/tooling/opfs-cache.js +168 -32
- package/src/tooling-exports/storage.d.ts +1 -1
- package/src/tooling-exports/storage.js +1 -1
- package/src/types/gpu.d.ts +25 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +1593 -0
- package/tests/kernels/browser/test-page.js +2629 -0
- package/tests/kernels/harness/benchmark.js +225 -0
- package/tests/kernels/harness/buffer-utils.js +88 -0
- package/tests/kernels/harness/tolerance.js +174 -0
- package/tests/kernels/reference/adam.js +15 -0
- package/tests/kernels/reference/attention.js +158 -0
- package/tests/kernels/reference/backward.js +96 -0
- package/tests/kernels/reference/bias-add.js +20 -0
- package/tests/kernels/reference/conv2d.js +58 -0
- package/tests/kernels/reference/cross-entropy.js +26 -0
- package/tests/kernels/reference/dequant.js +312 -0
- package/tests/kernels/reference/embed.js +15 -0
- package/tests/kernels/reference/gather.js +61 -0
- package/tests/kernels/reference/gelu.js +37 -0
- package/tests/kernels/reference/groupnorm.js +36 -0
- package/tests/kernels/reference/index.js +89 -0
- package/tests/kernels/reference/layernorm.js +22 -0
- package/tests/kernels/reference/matmul.js +57 -0
- package/tests/kernels/reference/modulate.js +24 -0
- package/tests/kernels/reference/moe-gather.js +83 -0
- package/tests/kernels/reference/pixel-shuffle.js +20 -0
- package/tests/kernels/reference/residual.js +55 -0
- package/tests/kernels/reference/rmsnorm.js +52 -0
- package/tests/kernels/reference/rope.js +79 -0
- package/tests/kernels/reference/sample.js +97 -0
- package/tests/kernels/reference/scatter-add.js +58 -0
- package/tests/kernels/reference/silu.js +52 -0
- package/tests/kernels/reference/softmax.js +86 -0
- package/tests/kernels/reference/split-qkv.js +58 -0
- package/tests/kernels/reference/topk.js +88 -0
- package/tests/kernels/reference/upsample2d.js +37 -0
- package/BRANDING.md +0 -14
- package/SECURITY.md +0 -19
- package/assets/doppler-runtime-map.svg +0 -82
- package/src/bootstrap.js +0 -30
- package/src/cli/doppler-cli.d.ts +0 -46
- package/src/cli/doppler-quickstart.d.ts +0 -45
- package/src/cli/doppler-serve.d.ts +0 -210
- package/src/cli/runtime-profiles.d.ts +0 -40
- package/src/client/doppler-api.browser.d.ts +0 -1
- package/src/client/doppler-api.browser.js +0 -25
- package/src/client/doppler-registry.d.ts +0 -54
- package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
- package/src/client/runtime/source-runtime.d.ts +0 -26
- package/src/client/runtime/storage.d.ts +0 -5
- package/src/config/README.md +0 -61
- package/src/config/conversion/README.md +0 -62
- package/src/config/goal-completion-matrix.json +0 -552
- package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
- package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
- package/src/config/kernels/moe/mixtral.paths.json +0 -46
- package/src/config/kernels/registry.d.ts +0 -51
- package/src/config/merge-contract-check.d.ts +0 -16
- package/src/config/merge-contract-check.js +0 -319
- package/src/config/merge-helpers.d.ts +0 -60
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
- package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
- package/src/config/param-categories.d.ts +0 -8
- package/src/config/param-validator.d.ts +0 -33
- package/src/config/platforms/loader.d.ts +0 -38
- package/src/config/quantization-contract-check.d.ts +0 -12
- package/src/config/quantization-contract-check.js +0 -91
- package/src/config/runtime/README.md +0 -28
- package/src/config/runtime-merge.d.ts +0 -5
- package/src/config/schema/claim-evidence-contract.schema.json +0 -93
- package/src/config/schema/command-surface-contract.schema.json +0 -81
- package/src/config/schema/goal-completion-matrix.schema.json +0 -223
- package/src/config/schema/model-artifact-contract.schema.json +0 -119
- package/src/config/schema/platform.schema.js +0 -1
- package/src/config/schema/policy-schema-registry.json +0 -31
- package/src/config/schema/subsystem-support-contract.schema.json +0 -48
- package/src/config/support-tiers/subsystems.json +0 -502
- package/src/config/supported-operations.d.ts +0 -14
- package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
- package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
- package/src/converter/conversion-plan.d.ts +0 -56
- package/src/converter/index.d.ts +0 -98
- package/src/converter/index.js +0 -59
- package/src/converter/parsers/diffusion.d.ts +0 -50
- package/src/converter/parsers/gguf.d.ts +0 -31
- package/src/converter/parsers/index.d.ts +0 -21
- package/src/converter/parsers/index.js +0 -12
- package/src/converter/parsers/transformer.d.ts +0 -31
- package/src/converter/quantization-info.d.ts +0 -38
- package/src/converter/rope-config.d.ts +0 -16
- package/src/converter/tokenizer-utils.d.ts +0 -12
- package/src/debug/reference/README.md +0 -114
- package/src/debug/reference/hf_attn_debug.py +0 -114
- package/src/debug/reference/hf_embed_check.py +0 -89
- package/src/debug/reference/hf_layer_out.py +0 -100
- package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
- package/src/debug/reference/hf_rope_check.py +0 -116
- package/src/debug/reference/hf_weights.py +0 -75
- package/src/errors/doppler-error.d.ts +0 -20
- package/src/experimental/bridge/extension/background.d.ts +0 -14
- package/src/experimental/bridge/extension/background.js +0 -168
- package/src/experimental/bridge/extension/manifest.json +0 -34
- package/src/experimental/bridge/extension-client.d.ts +0 -114
- package/src/experimental/bridge/index.d.ts +0 -69
- package/src/experimental/bridge/node-bridge.js +0 -24
- package/src/experimental/bridge/protocol.d.ts +0 -96
- package/src/experimental/browser/gguf-importer.d.ts +0 -136
- package/src/experimental/browser/gguf-importer.js +0 -532
- package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
- package/src/experimental/browser/quantization.d.ts +0 -74
- package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
- package/src/experimental/browser/shard-io-browser.d.ts +0 -57
- package/src/experimental/browser/shard-io-browser.js +0 -89
- package/src/experimental/browser/tensor-source-download.d.ts +0 -27
- package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
- package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
- package/src/experimental/distribution/p2p-transport.js +0 -75
- package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
- package/src/experimental/hotswap/manifest.d.ts +0 -42
- package/src/experimental/hotswap/model-swap.js +0 -42
- package/src/experimental/hotswap/runtime.d.ts +0 -31
- package/src/experimental/training/README.md +0 -164
- package/src/experimental/training/artifacts.d.ts +0 -160
- package/src/experimental/training/attention-backward.d.ts +0 -30
- package/src/experimental/training/attention-forward.d.ts +0 -22
- package/src/experimental/training/attention-forward.js +0 -82
- package/src/experimental/training/autograd.d.ts +0 -51
- package/src/experimental/training/backward-kernels.js +0 -39
- package/src/experimental/training/checkpoint-watch.d.ts +0 -8
- package/src/experimental/training/checkpoint.d.ts +0 -36
- package/src/experimental/training/clip.d.ts +0 -9
- package/src/experimental/training/dataloader.d.ts +0 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
- package/src/experimental/training/datasets/gepa-frontier.js +0 -220
- package/src/experimental/training/datasets/index.d.ts +0 -24
- package/src/experimental/training/datasets/index.js +0 -13
- package/src/experimental/training/datasets/jsonl.d.ts +0 -11
- package/src/experimental/training/datasets/reploid.d.ts +0 -3
- package/src/experimental/training/datasets/reploid.js +0 -36
- package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
- package/src/experimental/training/datasets/teacher-traces.js +0 -279
- package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
- package/src/experimental/training/datasets/token-batch.d.ts +0 -21
- package/src/experimental/training/datasets/token-batch.js +0 -52
- package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
- package/src/experimental/training/datasets/translation-pairs.js +0 -49
- package/src/experimental/training/distillation/artifacts.d.ts +0 -71
- package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
- package/src/experimental/training/distillation/dataset.d.ts +0 -59
- package/src/experimental/training/distillation/eval.d.ts +0 -34
- package/src/experimental/training/distillation/index.d.ts +0 -29
- package/src/experimental/training/distillation/runtime.d.ts +0 -20
- package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
- package/src/experimental/training/distillation/stage-a.d.ts +0 -45
- package/src/experimental/training/distillation/stage-b.d.ts +0 -24
- package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
- package/src/experimental/training/distillation/suite-data.d.ts +0 -45
- package/src/experimental/training/export.d.ts +0 -41
- package/src/experimental/training/index.d.ts +0 -74
- package/src/experimental/training/index.js +0 -63
- package/src/experimental/training/lora-pipeline.d.ts +0 -183
- package/src/experimental/training/lora.d.ts +0 -19
- package/src/experimental/training/loss-scaling.d.ts +0 -21
- package/src/experimental/training/loss.d.ts +0 -10
- package/src/experimental/training/objectives/base.d.ts +0 -58
- package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
- package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
- package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
- package/src/experimental/training/objectives/index.d.ts +0 -12
- package/src/experimental/training/objectives/index.js +0 -6
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
- package/src/experimental/training/operator-agent-eval.d.ts +0 -65
- package/src/experimental/training/operator-artifacts.d.ts +0 -62
- package/src/experimental/training/operator-command.d.ts +0 -5
- package/src/experimental/training/operator-eval.d.ts +0 -48
- package/src/experimental/training/operator-scoreboard.d.ts +0 -5
- package/src/experimental/training/optimizer.d.ts +0 -22
- package/src/experimental/training/runner.d.ts +0 -249
- package/src/experimental/training/suite.d.ts +0 -299
- package/src/experimental/training/tensor-factory.d.ts +0 -9
- package/src/experimental/training/trainer.d.ts +0 -89
- package/src/experimental/training/ul_dataset.d.ts +0 -47
- package/src/experimental/training/ul_schedule.d.ts +0 -6
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
- package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
- package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
- package/src/experimental/training/workload-packs/registry.json +0 -278
- package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
- package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
- package/src/experimental/training/workloads.d.ts +0 -201
- package/src/formats/gguf/types.d.ts +0 -74
- package/src/formats/index.d.ts +0 -75
- package/src/formats/index.js +0 -19
- package/src/formats/litert/types.d.ts +0 -74
- package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
- package/src/formats/rdrr/distributed/validation.d.ts +0 -23
- package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
- package/src/formats/safetensors/types.d.ts +0 -67
- package/src/formats/tflite/types.d.ts +0 -61
- package/src/formats/tokenizer/index.d.ts +0 -5
- package/src/formats/tokenizer/index.js +0 -3
- package/src/formats/tokenizer/types.d.ts +0 -9
- package/src/gpu/kernel-runtime.d.ts +0 -13
- package/src/gpu/kernel-selection-cache.d.ts +0 -13
- package/src/gpu/kernel-selection-log.d.ts +0 -4
- package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
- package/src/gpu/kernel-tuner/cache.d.ts +0 -55
- package/src/gpu/kernel-tuner/index.d.ts +0 -59
- package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
- package/src/gpu/kernel-tuner/types.d.ts +0 -101
- package/src/gpu/kernel-tuner/types.js +0 -4
- package/src/gpu/kernel-tuner.d.ts +0 -33
- package/src/gpu/kernels/README.md +0 -166
- package/src/gpu/kernels/check-finiteness.d.ts +0 -15
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
- package/src/gpu/kernels/check-stop.d.ts +0 -31
- package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
- package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
- package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
- package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
- package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
- package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
- package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
- package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
- package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
- package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
- package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
- package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
- package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
- package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
- package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
- package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
- package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
- package/src/gpu/kernels/constants.d.ts +0 -129
- package/src/gpu/kernels/dispatch.d.ts +0 -72
- package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
- package/src/gpu/kernels/kernel-base.d.ts +0 -33
- package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
- package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
- package/src/gpu/kernels/matmul-selection.d.ts +0 -87
- package/src/gpu/kernels/rep-penalty.d.ts +0 -21
- package/src/gpu/kernels/rule-matcher.d.ts +0 -30
- package/src/gpu/kernels/rule-registry.d.ts +0 -7
- package/src/gpu/kernels/types.js +0 -4
- package/src/gpu/perf-guards.d.ts +0 -13
- package/src/gpu/profiler.d.ts +0 -93
- package/src/gpu/readback-utils.d.ts +0 -16
- package/src/gpu/readback-utils.js +0 -49
- package/src/gpu/submit-probe.d.ts +0 -12
- package/src/index-browser.d.ts +0 -2
- package/src/index-browser.js +0 -2
- package/src/inference/README.md +0 -593
- package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
- package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
- package/src/inference/browser-harness-model-helpers.d.ts +0 -10
- package/src/inference/browser-harness-report-helpers.d.ts +0 -7
- package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
- package/src/inference/browser-harness-text-helpers.d.ts +0 -152
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
- package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
- package/src/inference/network-evolution.js +0 -79
- package/src/inference/pipelines/context.d.ts +0 -21
- package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
- package/src/inference/pipelines/diffusion/index.d.ts +0 -3
- package/src/inference/pipelines/diffusion/index.js +0 -3
- package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
- package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
- package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
- package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
- package/src/inference/pipelines/diffusion/types.js +0 -1
- package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
- package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
- package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
- package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
- package/src/inference/pipelines/dream/pipeline.js +0 -8
- package/src/inference/pipelines/energy/index.d.ts +0 -1
- package/src/inference/pipelines/energy/index.js +0 -1
- package/src/inference/pipelines/energy/types.js +0 -1
- package/src/inference/pipelines/energy-head/index.d.ts +0 -6
- package/src/inference/pipelines/energy-head/index.js +0 -6
- package/src/inference/pipelines/factory.d.ts +0 -10
- package/src/inference/pipelines/index.d.ts +0 -22
- package/src/inference/pipelines/index.js +0 -19
- package/src/inference/pipelines/registry.d.ts +0 -15
- package/src/inference/pipelines/rng.d.ts +0 -2
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
- package/src/inference/pipelines/text/attention/index.d.ts +0 -24
- package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
- package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
- package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
- package/src/inference/pipelines/text/attention/record.d.ts +0 -36
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
- package/src/inference/pipelines/text/attention/run.d.ts +0 -38
- package/src/inference/pipelines/text/attention/types.d.ts +0 -125
- package/src/inference/pipelines/text/attention.d.ts +0 -23
- package/src/inference/pipelines/text/attention.js +0 -12
- package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
- package/src/inference/pipelines/text/buffer-types.js +0 -4
- package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
- package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
- package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
- package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
- package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
- package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
- package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
- package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
- package/src/inference/pipelines/text/embed.d.ts +0 -69
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
- package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
- package/src/inference/pipelines/text/failure-reduction.js +0 -180
- package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
- package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
- package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
- package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
- package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
- package/src/inference/pipelines/text/ffn.d.ts +0 -31
- package/src/inference/pipelines/text/ffn.js +0 -18
- package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
- package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
- package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
- package/src/inference/pipelines/text/index.d.ts +0 -5
- package/src/inference/pipelines/text/index.js +0 -6
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
- package/src/inference/pipelines/text/layer.d.ts +0 -118
- package/src/inference/pipelines/text/load-errors.d.ts +0 -21
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
- package/src/inference/pipelines/text/logits/types.js +0 -4
- package/src/inference/pipelines/text/logits.d.ts +0 -27
- package/src/inference/pipelines/text/logits.js +0 -16
- package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
- package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
- package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
- package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
- package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
- package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
- package/src/inference/pipelines/text/operator-events.d.ts +0 -125
- package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
- package/src/inference/pipelines/text/ops.d.ts +0 -222
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
- package/src/inference/pipelines/text/probes.d.ts +0 -28
- package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
- package/src/inference/pipelines/text/stage-names.d.ts +0 -88
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
- package/src/inference/pipelines/text/types.js +0 -4
- package/src/inference/tokenizers/base.d.ts +0 -44
- package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
- package/src/inference/tokenizers/bpe.d.ts +0 -27
- package/src/inference/tokenizers/bundled.d.ts +0 -67
- package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
- package/src/inference/tokenizers/types.js +0 -7
- package/src/loader/embedding-limit-preflight.d.ts +0 -31
- package/src/loader/per-layer-input-loader.d.ts +0 -30
- package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
- package/src/loader/tensors/source-transform.d.ts +0 -65
- package/src/memory/address-table.d.ts +0 -96
- package/src/rules/execution-rules-contract-check.d.ts +0 -21
- package/src/rules/layer-pattern-contract-check.d.ts +0 -17
- package/src/rules/rule-registry.d.ts +0 -77
- package/src/storage/artifact-storage-context.d.ts +0 -63
- package/src/storage/backends/idb-store.d.ts +0 -52
- package/src/storage/backends/memory-store.d.ts +0 -36
- package/src/storage/backends/opfs-store.d.ts +0 -41
- package/src/storage/blake3.d.ts +0 -17
- package/src/storage/download-types.d.ts +0 -157
- package/src/storage/downloader.d.ts +0 -103
- package/src/storage/index.d.ts +0 -274
- package/src/storage/index.js +0 -201
- package/src/storage/preflight.d.ts +0 -144
- package/src/storage/preflight.js +0 -360
- package/src/storage/quickstart-downloader.d.ts +0 -157
- package/src/storage/quickstart-downloader.js +0 -289
- package/src/storage/source-artifact-store.d.ts +0 -59
- package/src/sw.js +0 -184
- package/src/tooling/command-api-constants.d.ts +0 -9
- package/src/tooling/command-api-family-normalizers.d.ts +0 -10
- package/src/tooling/command-api-helpers.d.ts +0 -23
- package/src/tooling/command-runner-shared.d.ts +0 -89
- package/src/tooling/conversion-config-materializer.d.ts +0 -14
- package/src/tooling/conversion-config-materializer.js +0 -96
- package/src/tooling/diagnose-runner.d.ts +0 -28
- package/src/tooling/hf-registry-utils.d.ts +0 -55
- package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
- package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
- package/src/tooling/kernel-path-builder/index.d.ts +0 -65
- package/src/tooling/kernel-path-builder/index.js +0 -1361
- package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
- package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
- package/src/tooling/lean-execution-contract-runner.js +0 -158
- package/src/tooling/lean-execution-contract.d.ts +0 -16
- package/src/tooling/lean-execution-contract.js +0 -81
- package/src/tooling/litert-package-runtime.d.ts +0 -46
- package/src/tooling/node-convert-worker-pool.d.ts +0 -16
- package/src/tooling/node-convert-worker.d.ts +0 -1
- package/src/tooling/node-converter.d.ts +0 -1
- package/src/tooling/node-file-fetch.d.ts +0 -1
- package/src/tooling/node-source-runtime.d.ts +0 -24
- package/src/tooling/node-webgpu.d.ts +0 -17
- package/src/tooling/precision-replay-math.js +0 -174
- package/src/tooling/program-bundle-host.d.ts +0 -16
- package/src/tooling/program-bundle-host.js +0 -20
- package/src/tooling/runtime-input-composition.d.ts +0 -37
- package/src/tooling/source-package-profiles.d.ts +0 -50
- package/src/tooling/source-runtime-converter-config.d.ts +0 -19
- package/src/tooling/source-runtime-materializer.d.ts +0 -6
- package/src/tooling/source-runtime-materializer.js +0 -89
- package/src/tooling-exports.browser.d.ts +0 -7
- package/src/types/chrome.d.ts +0 -36
- package/src/types/chrome.js +0 -1
- package/src/types/index.d.ts +0 -1
- package/src/types/index.js +0 -1
- package/src/utils/array-buffer.d.ts +0 -4
- package/src/utils/canonical-hash.d.ts +0 -7
- package/src/utils/clone-json.d.ts +0 -1
- package/src/utils/hf-resolve-url.d.ts +0 -16
- package/src/utils/index.d.ts +0 -10
- package/src/utils/index.js +0 -10
- package/src/utils/load-json.d.ts +0 -5
- package/src/utils/plain-object.d.ts +0 -1
- package/src/utils/sha256.d.ts +0 -5
- package/src/utils/stable-sort-object.d.ts +0 -1
|
@@ -104,6 +104,39 @@ function getSplitQKVRMSNormRoPEQKRunner(recorder) {
|
|
|
104
104
|
);
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
+
async function projectFusedQGate({
|
|
108
|
+
runMatmulForMode,
|
|
109
|
+
projectionInput,
|
|
110
|
+
layerWeights,
|
|
111
|
+
numTokens,
|
|
112
|
+
qSize,
|
|
113
|
+
hiddenSize,
|
|
114
|
+
layerIdx,
|
|
115
|
+
kernelPath,
|
|
116
|
+
projectionOutputDtype,
|
|
117
|
+
matmulDebug,
|
|
118
|
+
executionPolicies,
|
|
119
|
+
fusedNormWeight,
|
|
120
|
+
fusedNormEps,
|
|
121
|
+
fusedNormOffset,
|
|
122
|
+
}) {
|
|
123
|
+
if (!layerWeights?.qGateProj) {
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
return runMatmulForMode(projectionInput, layerWeights.qGateProj, numTokens, qSize, hiddenSize, {
|
|
127
|
+
transposeB: 'auto',
|
|
128
|
+
role: 'q_proj',
|
|
129
|
+
layerIdx,
|
|
130
|
+
kernelPath,
|
|
131
|
+
outputDtype: projectionOutputDtype,
|
|
132
|
+
matmulDebug,
|
|
133
|
+
executionPolicies,
|
|
134
|
+
normWeight: fusedNormWeight,
|
|
135
|
+
rmsNormEps: fusedNormEps,
|
|
136
|
+
rmsNormOffset: fusedNormOffset,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
|
|
107
140
|
function releaseOwnedWeightBuffer(layerWeight, resolvedWeightBuffer, releaseTemporary) {
|
|
108
141
|
if (isGpuBufferInstance(layerWeight) || isWeightBuffer(layerWeight)) {
|
|
109
142
|
return;
|
|
@@ -564,6 +597,7 @@ export async function projectAttentionQKV({
|
|
|
564
597
|
}) {
|
|
565
598
|
const runMatmulForMode = getMatmulRunner(recorder);
|
|
566
599
|
const runSplitForMode = getSplitRunner(recorder);
|
|
600
|
+
const runSplitQGForMode = getSplitQGRunner(recorder);
|
|
567
601
|
const runSplitQKVRMSNormQKForMode = getSplitQKVRMSNormQKRunner(recorder);
|
|
568
602
|
const runSplitQKVRMSNormRoPEQKForMode = getSplitQKVRMSNormRoPEQKRunner(recorder);
|
|
569
603
|
const reuseSharedKV = sharedKTensor != null || sharedVTensor != null;
|
|
@@ -605,6 +639,21 @@ export async function projectAttentionQKV({
|
|
|
605
639
|
if (useFusedQKV && layerWeights.qkvProj && layerWeights.qkvSizes) {
|
|
606
640
|
const [qSizeFused, kSizeFused, vSizeFused] = layerWeights.qkvSizes;
|
|
607
641
|
const qkvSizeTotal = qSizeFused + kSizeFused + vSizeFused;
|
|
642
|
+
const qProjectionSize = numHeads * headDim;
|
|
643
|
+
const qProjectionContainsGate = attentionOutputGate === true;
|
|
644
|
+
const hasSeparateGateProjection = qProjectionContainsGate && layerWeights.qGateProj;
|
|
645
|
+
if (hasSeparateGateProjection && qSizeFused !== qProjectionSize) {
|
|
646
|
+
throw new Error(
|
|
647
|
+
`Fused QKV for attention-output-gate layer ${layerIdx} with qGateProj must store ` +
|
|
648
|
+
`Q-only first slice with ${qProjectionSize} columns; got ${qSizeFused}.`
|
|
649
|
+
);
|
|
650
|
+
}
|
|
651
|
+
if (qProjectionContainsGate && !hasSeparateGateProjection && qSizeFused !== qProjectionSize * 2) {
|
|
652
|
+
throw new Error(
|
|
653
|
+
`Fused QKV for attention-output-gate layer ${layerIdx} must store Q+gate first slice ` +
|
|
654
|
+
`with ${qProjectionSize * 2} columns; got ${qSizeFused}.`
|
|
655
|
+
);
|
|
656
|
+
}
|
|
608
657
|
let qkvTensor = null;
|
|
609
658
|
let qNormBuf = null;
|
|
610
659
|
let kNormBuf = null;
|
|
@@ -629,6 +678,7 @@ export async function projectAttentionQKV({
|
|
|
629
678
|
? { ...qkNormRoPEFusion, headDim }
|
|
630
679
|
: null;
|
|
631
680
|
const canFuseSplitQKNormAndRoPE = qkNormRoPEFusion?.enabled === true
|
|
681
|
+
&& (!qProjectionContainsGate || hasSeparateGateProjection)
|
|
632
682
|
&& qkNormRoPEFusion.projectionDiagnosticsEnabled !== true
|
|
633
683
|
&& qkNormRoPEFusion.skipKNorm !== true
|
|
634
684
|
&& qkNormRoPEFusion.allowUnitQKNorm !== true
|
|
@@ -669,12 +719,44 @@ export async function projectAttentionQKV({
|
|
|
669
719
|
);
|
|
670
720
|
releaseTemporary(qkvTensor.buffer);
|
|
671
721
|
qkvTensor = null;
|
|
722
|
+
let qGateTensor = null;
|
|
723
|
+
if (hasSeparateGateProjection) {
|
|
724
|
+
try {
|
|
725
|
+
qGateTensor = await projectFusedQGate({
|
|
726
|
+
runMatmulForMode,
|
|
727
|
+
projectionInput,
|
|
728
|
+
layerWeights,
|
|
729
|
+
numTokens,
|
|
730
|
+
qSize: qProjectionSize,
|
|
731
|
+
hiddenSize,
|
|
732
|
+
layerIdx,
|
|
733
|
+
kernelPath,
|
|
734
|
+
projectionOutputDtype,
|
|
735
|
+
matmulDebug,
|
|
736
|
+
executionPolicies,
|
|
737
|
+
fusedNormWeight,
|
|
738
|
+
fusedNormEps,
|
|
739
|
+
fusedNormOffset,
|
|
740
|
+
});
|
|
741
|
+
} catch (error) {
|
|
742
|
+
if (fused.Q?.buffer) {
|
|
743
|
+
releaseTemporary(fused.Q.buffer);
|
|
744
|
+
}
|
|
745
|
+
if (fused.K?.buffer) {
|
|
746
|
+
releaseTemporary(fused.K.buffer);
|
|
747
|
+
}
|
|
748
|
+
if (fused.V?.buffer) {
|
|
749
|
+
releaseTemporary(fused.V.buffer);
|
|
750
|
+
}
|
|
751
|
+
throw error;
|
|
752
|
+
}
|
|
753
|
+
}
|
|
672
754
|
if (onFusedQKV) {
|
|
673
755
|
onFusedQKV({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize: qkvSizeTotal });
|
|
674
756
|
}
|
|
675
757
|
return {
|
|
676
758
|
qTensor: fused.Q,
|
|
677
|
-
qGateTensor
|
|
759
|
+
qGateTensor,
|
|
678
760
|
kTensor: fused.K,
|
|
679
761
|
vTensor: fused.V,
|
|
680
762
|
usedFusedQKV: true,
|
|
@@ -686,6 +768,7 @@ export async function projectAttentionQKV({
|
|
|
686
768
|
}
|
|
687
769
|
}
|
|
688
770
|
const canFuseSplitAndQKNorm = qkNormFusion?.enabled === true
|
|
771
|
+
&& (!qProjectionContainsGate || hasSeparateGateProjection)
|
|
689
772
|
&& qkNormFusion.projectionDiagnosticsEnabled !== true
|
|
690
773
|
&& qkNormFusion.skipKNorm !== true
|
|
691
774
|
&& qkNormFusion.allowUnitQKNorm !== true
|
|
@@ -717,12 +800,44 @@ export async function projectAttentionQKV({
|
|
|
717
800
|
);
|
|
718
801
|
releaseTemporary(qkvTensor.buffer);
|
|
719
802
|
qkvTensor = null;
|
|
803
|
+
let qGateTensor = null;
|
|
804
|
+
if (hasSeparateGateProjection) {
|
|
805
|
+
try {
|
|
806
|
+
qGateTensor = await projectFusedQGate({
|
|
807
|
+
runMatmulForMode,
|
|
808
|
+
projectionInput,
|
|
809
|
+
layerWeights,
|
|
810
|
+
numTokens,
|
|
811
|
+
qSize: qProjectionSize,
|
|
812
|
+
hiddenSize,
|
|
813
|
+
layerIdx,
|
|
814
|
+
kernelPath,
|
|
815
|
+
projectionOutputDtype,
|
|
816
|
+
matmulDebug,
|
|
817
|
+
executionPolicies,
|
|
818
|
+
fusedNormWeight,
|
|
819
|
+
fusedNormEps,
|
|
820
|
+
fusedNormOffset,
|
|
821
|
+
});
|
|
822
|
+
} catch (error) {
|
|
823
|
+
if (fused.Q?.buffer) {
|
|
824
|
+
releaseTemporary(fused.Q.buffer);
|
|
825
|
+
}
|
|
826
|
+
if (fused.K?.buffer) {
|
|
827
|
+
releaseTemporary(fused.K.buffer);
|
|
828
|
+
}
|
|
829
|
+
if (fused.V?.buffer) {
|
|
830
|
+
releaseTemporary(fused.V.buffer);
|
|
831
|
+
}
|
|
832
|
+
throw error;
|
|
833
|
+
}
|
|
834
|
+
}
|
|
720
835
|
if (onFusedQKV) {
|
|
721
836
|
onFusedQKV({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize: qkvSizeTotal });
|
|
722
837
|
}
|
|
723
838
|
return {
|
|
724
839
|
qTensor: fused.Q,
|
|
725
|
-
qGateTensor
|
|
840
|
+
qGateTensor,
|
|
726
841
|
kTensor: fused.K,
|
|
727
842
|
vTensor: fused.V,
|
|
728
843
|
usedFusedQKV: true,
|
|
@@ -740,12 +855,67 @@ export async function projectAttentionQKV({
|
|
|
740
855
|
vSize: vSizeFused,
|
|
741
856
|
});
|
|
742
857
|
releaseTemporary(qkvTensor.buffer);
|
|
858
|
+
qkvTensor = null;
|
|
859
|
+
let qTensor = split.Q;
|
|
860
|
+
let qGateTensor = null;
|
|
861
|
+
if (hasSeparateGateProjection) {
|
|
862
|
+
try {
|
|
863
|
+
qGateTensor = await projectFusedQGate({
|
|
864
|
+
runMatmulForMode,
|
|
865
|
+
projectionInput,
|
|
866
|
+
layerWeights,
|
|
867
|
+
numTokens,
|
|
868
|
+
qSize: qProjectionSize,
|
|
869
|
+
hiddenSize,
|
|
870
|
+
layerIdx,
|
|
871
|
+
kernelPath,
|
|
872
|
+
projectionOutputDtype,
|
|
873
|
+
matmulDebug,
|
|
874
|
+
executionPolicies,
|
|
875
|
+
fusedNormWeight,
|
|
876
|
+
fusedNormEps,
|
|
877
|
+
fusedNormOffset,
|
|
878
|
+
});
|
|
879
|
+
} catch (error) {
|
|
880
|
+
if (split.Q?.buffer) {
|
|
881
|
+
releaseTemporary(split.Q.buffer);
|
|
882
|
+
}
|
|
883
|
+
if (split.K?.buffer) {
|
|
884
|
+
releaseTemporary(split.K.buffer);
|
|
885
|
+
}
|
|
886
|
+
if (split.V?.buffer) {
|
|
887
|
+
releaseTemporary(split.V.buffer);
|
|
888
|
+
}
|
|
889
|
+
throw error;
|
|
890
|
+
}
|
|
891
|
+
} else if (qProjectionContainsGate) {
|
|
892
|
+
const qgTensor = split.Q;
|
|
893
|
+
try {
|
|
894
|
+
const qg = await runSplitQGForMode(qgTensor, {
|
|
895
|
+
numTokens,
|
|
896
|
+
numHeads,
|
|
897
|
+
headDim,
|
|
898
|
+
});
|
|
899
|
+
qTensor = qg.Q;
|
|
900
|
+
qGateTensor = qg.G;
|
|
901
|
+
releaseTemporary(qgTensor.buffer);
|
|
902
|
+
} catch (error) {
|
|
903
|
+
releaseTemporary(qgTensor.buffer);
|
|
904
|
+
if (split.K?.buffer) {
|
|
905
|
+
releaseTemporary(split.K.buffer);
|
|
906
|
+
}
|
|
907
|
+
if (split.V?.buffer) {
|
|
908
|
+
releaseTemporary(split.V.buffer);
|
|
909
|
+
}
|
|
910
|
+
throw error;
|
|
911
|
+
}
|
|
912
|
+
}
|
|
743
913
|
if (onFusedQKV) {
|
|
744
914
|
onFusedQKV({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize: qkvSizeTotal });
|
|
745
915
|
}
|
|
746
916
|
return {
|
|
747
|
-
qTensor
|
|
748
|
-
qGateTensor
|
|
917
|
+
qTensor,
|
|
918
|
+
qGateTensor,
|
|
749
919
|
kTensor: split.K,
|
|
750
920
|
vTensor: split.V,
|
|
751
921
|
usedFusedQKV: true,
|
|
@@ -62,6 +62,7 @@ import {
|
|
|
62
62
|
isAttentionKvDtypeExplicit,
|
|
63
63
|
} from './precision-contract.js';
|
|
64
64
|
import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
|
|
65
|
+
import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
|
|
65
66
|
|
|
66
67
|
const ATTENTION_DTYPE_LOGGED = new Set();
|
|
67
68
|
|
|
@@ -390,20 +391,23 @@ export async function recordLayerAttentionGPU(
|
|
|
390
391
|
const runtimeSession = getRuntimeConfig()?.inference?.session;
|
|
391
392
|
const qkNormFusionFlag = runtimeSession?.useFusedQKVSplitQKNorm === true;
|
|
392
393
|
const qkNormRoPEFusionFlag = runtimeSession?.useFusedQKVSplitQKNormRoPE === true;
|
|
393
|
-
const
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
394
|
+
const skipKVCacheWrites = state.skipKVCacheWrites === true;
|
|
395
|
+
const directF16KVCacheWrite = skipKVCacheWrites
|
|
396
|
+
? null
|
|
397
|
+
: resolveDirectF16KVCacheWrite({
|
|
398
|
+
state,
|
|
399
|
+
layerIdx,
|
|
400
|
+
currentSeqLen,
|
|
401
|
+
numTokens,
|
|
402
|
+
numKVHeads,
|
|
403
|
+
headDim,
|
|
404
|
+
reusesSharedKV,
|
|
405
|
+
storeSharedKV,
|
|
406
|
+
diffusionGemmaDecoder,
|
|
407
|
+
valueNorm: config.valueNorm === true,
|
|
408
|
+
diagnosticsEnabled: qkNormRoPEDiagnostics,
|
|
409
|
+
disableRoPE,
|
|
410
|
+
});
|
|
407
411
|
let kvCacheWriteFused = false;
|
|
408
412
|
({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey, qkNormApplied, ropeApplied, kvCacheWriteFused } = await projectAttentionQKV({
|
|
409
413
|
recorder,
|
|
@@ -649,7 +653,7 @@ export async function recordLayerAttentionGPU(
|
|
|
649
653
|
}
|
|
650
654
|
|
|
651
655
|
// 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
|
|
652
|
-
if (!diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
|
|
656
|
+
if (!skipKVCacheWrites && !diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
|
|
653
657
|
if (kvCacheWriteFused) {
|
|
654
658
|
state.kvCache.recordF16UpdateAlreadyWrittenFromGPU(layerIdx, currentSeqLen, numTokens, tokenIds);
|
|
655
659
|
} else if (state.kvCache.kvDtype === 'f16') {
|
|
@@ -741,7 +745,9 @@ export async function recordLayerAttentionGPU(
|
|
|
741
745
|
}
|
|
742
746
|
decoderKVState = kvState;
|
|
743
747
|
} else {
|
|
744
|
-
kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens
|
|
748
|
+
kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens, {
|
|
749
|
+
forceCurrentTensors: skipKVCacheWrites,
|
|
750
|
+
});
|
|
745
751
|
}
|
|
746
752
|
const dispatchConfig = {
|
|
747
753
|
layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
|
|
@@ -758,6 +764,7 @@ export async function recordLayerAttentionGPU(
|
|
|
758
764
|
cachedKDtype, cachedVDtype, cachedKTensor, cachedVTensor,
|
|
759
765
|
prefillFallbackNeedsCast, causalForAttention,
|
|
760
766
|
} = dispatchParams;
|
|
767
|
+
const attentionKernelPath = skipKVCacheWrites ? null : kernelPath;
|
|
761
768
|
|
|
762
769
|
// 5. Attention
|
|
763
770
|
|
|
@@ -767,6 +774,7 @@ export async function recordLayerAttentionGPU(
|
|
|
767
774
|
usedFusedQKV, qTensor, kTensor, vTensor,
|
|
768
775
|
));
|
|
769
776
|
const mergedSessionRec = getRuntimeConfig()?.inference?.session;
|
|
777
|
+
let attentionOutputGateFused = false;
|
|
770
778
|
|
|
771
779
|
const attentionKernelRunners = {
|
|
772
780
|
bdpa: async () => {
|
|
@@ -923,7 +931,18 @@ export async function recordLayerAttentionGPU(
|
|
|
923
931
|
// path in ./run.js.
|
|
924
932
|
const useFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useFlashPrefillAttention === true && numTokens > 1;
|
|
925
933
|
const useOrtFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useOrtFlashPrefillAttention === true && numTokens > 1;
|
|
926
|
-
|
|
934
|
+
const useOutputGateFusion = canUseAttentionOutputGateFusion({
|
|
935
|
+
session: mergedSessionRec,
|
|
936
|
+
qGateTensor,
|
|
937
|
+
numTokens,
|
|
938
|
+
numHeads,
|
|
939
|
+
headDim,
|
|
940
|
+
cachedKDtype,
|
|
941
|
+
cachedVDtype,
|
|
942
|
+
kernelPath: attentionKernelPath,
|
|
943
|
+
diffusionGemmaDecoder,
|
|
944
|
+
});
|
|
945
|
+
const result = await recordAttention(recorder, qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
|
|
927
946
|
seqLen: numTokens,
|
|
928
947
|
kvLen: kvState.kvLenForAttention,
|
|
929
948
|
numKVHeads,
|
|
@@ -939,10 +958,13 @@ export async function recordLayerAttentionGPU(
|
|
|
939
958
|
kvLayout: kvState.kvLayout,
|
|
940
959
|
kvPageTable: kvState.kvPageTable,
|
|
941
960
|
kvPageSize: kvState.kvPageSize,
|
|
942
|
-
kernelPath,
|
|
943
|
-
useFlashPrefill: useFlashPrefillRec,
|
|
944
|
-
useOrtFlashPrefill: useOrtFlashPrefillRec,
|
|
961
|
+
kernelPath: attentionKernelPath,
|
|
962
|
+
useFlashPrefill: skipKVCacheWrites ? false : useFlashPrefillRec,
|
|
963
|
+
useOrtFlashPrefill: skipKVCacheWrites ? false : useOrtFlashPrefillRec,
|
|
964
|
+
outputGate: useOutputGateFusion ? qGateTensor : null,
|
|
945
965
|
});
|
|
966
|
+
attentionOutputGateFused = result?.outputGateFused === true;
|
|
967
|
+
return result;
|
|
946
968
|
},
|
|
947
969
|
};
|
|
948
970
|
const runAttentionKernel = attentionKernelRunners[attentionKernelVariant];
|
|
@@ -980,13 +1002,14 @@ export async function recordLayerAttentionGPU(
|
|
|
980
1002
|
}
|
|
981
1003
|
|
|
982
1004
|
attnForProjection = attnOutput;
|
|
983
|
-
if (qGateTensor) {
|
|
1005
|
+
if (qGateTensor && !attentionOutputGateFused) {
|
|
984
1006
|
// Mirror run.js gate dispatch. Qwen3_5/Qwen 3.6 full attention applies
|
|
985
1007
|
// sigmoid(gate) even when the HF config surfaces `output_gate_type=swish`.
|
|
986
1008
|
const gateActivation = 'sigmoid';
|
|
987
1009
|
attnForProjection = await recordSiLU(recorder, attnOutput, {
|
|
988
1010
|
size: numTokens * numHeads * headDim,
|
|
989
1011
|
gate: qGateTensor,
|
|
1012
|
+
useVec4: (numTokens * numHeads * headDim) % 4 === 0,
|
|
990
1013
|
gateActivation,
|
|
991
1014
|
inputActivation: 'identity',
|
|
992
1015
|
swigluLimit: null,
|
|
@@ -66,6 +66,7 @@ import {
|
|
|
66
66
|
isAttentionKvDtypeExplicit,
|
|
67
67
|
} from './precision-contract.js';
|
|
68
68
|
import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
|
|
69
|
+
import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
|
|
69
70
|
|
|
70
71
|
const ATTENTION_DTYPE_LOGGED = new Set();
|
|
71
72
|
|
|
@@ -793,6 +794,8 @@ export async function runLayerAttentionGPU(
|
|
|
793
794
|
cachedKDtype, cachedVDtype, cachedKTensor, cachedVTensor,
|
|
794
795
|
prefillFallbackNeedsCast, causalForAttention,
|
|
795
796
|
} = dispatchParams;
|
|
797
|
+
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
798
|
+
let attentionOutputGateFused = false;
|
|
796
799
|
|
|
797
800
|
// 5. Attention (uses raw GPUBuffers)
|
|
798
801
|
|
|
@@ -991,9 +994,19 @@ export async function runLayerAttentionGPU(
|
|
|
991
994
|
// session (state.runtimeConfig.inference.session is a stale snapshot —
|
|
992
995
|
// see comment above at the rmsNorm fusion check). The kernel itself
|
|
993
996
|
// enforces remaining preconditions (head_dim=256, etc.).
|
|
994
|
-
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
995
997
|
const useFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useFlashPrefillAttention === true && numTokens > 1;
|
|
996
998
|
const useOrtFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useOrtFlashPrefillAttention === true && numTokens > 1;
|
|
999
|
+
const useOutputGateFusion = canUseAttentionOutputGateFusion({
|
|
1000
|
+
session: mergedSession,
|
|
1001
|
+
qGateTensor,
|
|
1002
|
+
numTokens,
|
|
1003
|
+
numHeads,
|
|
1004
|
+
headDim,
|
|
1005
|
+
cachedKDtype,
|
|
1006
|
+
cachedVDtype,
|
|
1007
|
+
kernelPath,
|
|
1008
|
+
diffusionGemmaDecoder,
|
|
1009
|
+
});
|
|
997
1010
|
const result = await runAttention(qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
|
|
998
1011
|
seqLen: numTokens,
|
|
999
1012
|
kvLen: kvState.kvLenForAttention,
|
|
@@ -1013,7 +1026,9 @@ export async function runLayerAttentionGPU(
|
|
|
1013
1026
|
kernelPath,
|
|
1014
1027
|
useFlashPrefill,
|
|
1015
1028
|
useOrtFlashPrefill,
|
|
1029
|
+
outputGate: useOutputGateFusion ? qGateTensor : null,
|
|
1016
1030
|
});
|
|
1031
|
+
attentionOutputGateFused = result?.outputGateFused === true;
|
|
1017
1032
|
if (prefillFallbackNeedsCast) {
|
|
1018
1033
|
if (kTensor.dtype !== 'f16') releaseBuffer(kForAttn.buffer);
|
|
1019
1034
|
if (vTensor.dtype !== 'f16') releaseBuffer(vForAttn.buffer);
|
|
@@ -1068,7 +1083,7 @@ export async function runLayerAttentionGPU(
|
|
|
1068
1083
|
}
|
|
1069
1084
|
|
|
1070
1085
|
attnForProjection = attnOutput;
|
|
1071
|
-
if (qGateTensor) {
|
|
1086
|
+
if (qGateTensor && !attentionOutputGateFused) {
|
|
1072
1087
|
// The Qwen3_5 reference (HF transformers) full-attention path
|
|
1073
1088
|
// applies sigmoid(gate) — NOT silu(gate) — to the attention
|
|
1074
1089
|
// output regardless of how the architecture's gate label is
|
|
@@ -1089,6 +1104,7 @@ export async function runLayerAttentionGPU(
|
|
|
1089
1104
|
attnForProjection = await runSiLU(attnOutput, {
|
|
1090
1105
|
size: numTokens * numHeads * headDim,
|
|
1091
1106
|
gate: qGateTensor,
|
|
1107
|
+
useVec4: (numTokens * numHeads * headDim) % 4 === 0,
|
|
1092
1108
|
gateActivation,
|
|
1093
1109
|
inputActivation: 'identity',
|
|
1094
1110
|
swigluLimit: null,
|
|
@@ -1125,7 +1141,6 @@ export async function runLayerAttentionGPU(
|
|
|
1125
1141
|
// Use fused o_proj + residual for decode when possible
|
|
1126
1142
|
// Note: dtype from WeightBuffer metadata (buffer-dtypes WeakMap removed)
|
|
1127
1143
|
const oProjDtype = getWeightDtype(oProjBuf);
|
|
1128
|
-
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
1129
1144
|
const canUseFused = selectRuleValue('inference', 'attention', 'useFusedOProjResidual', {
|
|
1130
1145
|
allowFusedResidual: shouldUseFusedMatmulResidual(numTokens),
|
|
1131
1146
|
hasResidual: Boolean(residualTensor),
|
|
@@ -114,14 +114,32 @@ export interface TensorInfo {
|
|
|
114
114
|
export interface ParsedSessionSettings {
|
|
115
115
|
prefillChunkSubmitMode: 'sync' | 'async' | null;
|
|
116
116
|
prefillTokenChunkSize: number | null;
|
|
117
|
+
skipEmbeddingKVCacheWrites: boolean | null;
|
|
117
118
|
useFlashPrefillAttention: boolean | null;
|
|
119
|
+
useLargeBatchF16F32FusedGateUp: boolean | null;
|
|
118
120
|
useWideTileQ4KPrefill: boolean | null;
|
|
119
121
|
useWideTileQ4KDecode: boolean | null;
|
|
120
122
|
useSandwichRMSNormPairFusion: boolean | null;
|
|
121
123
|
usePostFfnNextInputRMSNormPairFusion: boolean | null;
|
|
124
|
+
usePostAttnNormFusedGateUp: boolean | null;
|
|
125
|
+
useLinearAttentionABProjectionFusion: boolean | null;
|
|
126
|
+
useLinearAttentionQKVZProjectionFusion: boolean | null;
|
|
127
|
+
useLinearAttentionFusedDecodeCore: boolean | null;
|
|
128
|
+
useWideTileResidualFusion: boolean | null;
|
|
129
|
+
useFusedRmsnormWideTile: boolean | null;
|
|
122
130
|
useFusedQKVSplitQKNorm: boolean | null;
|
|
123
131
|
useFusedQKVSplitQKNormRoPE: boolean | null;
|
|
124
132
|
retainQ4KMaterialization: boolean | null;
|
|
133
|
+
lmHeadArgmaxQ4K: {
|
|
134
|
+
useFullBlockFastPath?: boolean;
|
|
135
|
+
colsPerWorkgroup?: number;
|
|
136
|
+
threadsPerCol?: number;
|
|
137
|
+
} | null;
|
|
138
|
+
attentionDecodeOnline: {
|
|
139
|
+
workgroupSize?: 128 | 256;
|
|
140
|
+
useDirectContiguousKVLayout?: boolean;
|
|
141
|
+
useOutputGateFusion?: boolean;
|
|
142
|
+
} | null;
|
|
125
143
|
}
|
|
126
144
|
|
|
127
145
|
export type Manifest = RuntimeModelContract;
|
|
@@ -223,10 +223,22 @@ function resolveSessionSettings(inferenceConfig, modelId) {
|
|
|
223
223
|
`"${String(tokenChunk)}"; expected null or a positive integer.`
|
|
224
224
|
);
|
|
225
225
|
}
|
|
226
|
+
const skipEmbeddingKVCacheWrites = session?.skipEmbeddingKVCacheWrites;
|
|
227
|
+
if (
|
|
228
|
+
skipEmbeddingKVCacheWrites !== undefined
|
|
229
|
+
&& skipEmbeddingKVCacheWrites !== null
|
|
230
|
+
&& typeof skipEmbeddingKVCacheWrites !== 'boolean'
|
|
231
|
+
) {
|
|
232
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.skipEmbeddingKVCacheWrites "${String(skipEmbeddingKVCacheWrites)}"; expected boolean.`);
|
|
233
|
+
}
|
|
226
234
|
const flash = session?.useFlashPrefillAttention;
|
|
227
235
|
if (flash !== undefined && flash !== null && typeof flash !== 'boolean') {
|
|
228
236
|
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFlashPrefillAttention "${String(flash)}"; expected boolean.`);
|
|
229
237
|
}
|
|
238
|
+
const largeBatchF16F32FusedGateUp = session?.useLargeBatchF16F32FusedGateUp;
|
|
239
|
+
if (largeBatchF16F32FusedGateUp !== undefined && largeBatchF16F32FusedGateUp !== null && typeof largeBatchF16F32FusedGateUp !== 'boolean') {
|
|
240
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useLargeBatchF16F32FusedGateUp "${String(largeBatchF16F32FusedGateUp)}"; expected boolean.`);
|
|
241
|
+
}
|
|
230
242
|
const wide = session?.useWideTileQ4KPrefill;
|
|
231
243
|
if (wide !== undefined && wide !== null && typeof wide !== 'boolean') {
|
|
232
244
|
throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileQ4KPrefill "${String(wide)}"; expected boolean.`);
|
|
@@ -243,6 +255,30 @@ function resolveSessionSettings(inferenceConfig, modelId) {
|
|
|
243
255
|
if (postFfnNextInputRmsNormPair !== undefined && postFfnNextInputRmsNormPair !== null && typeof postFfnNextInputRmsNormPair !== 'boolean') {
|
|
244
256
|
throw new Error(`Manifest "${modelId}" has invalid inference.session.usePostFfnNextInputRMSNormPairFusion "${String(postFfnNextInputRmsNormPair)}"; expected boolean.`);
|
|
245
257
|
}
|
|
258
|
+
const postAttnNormFusedGateUp = session?.usePostAttnNormFusedGateUp;
|
|
259
|
+
if (postAttnNormFusedGateUp !== undefined && postAttnNormFusedGateUp !== null && typeof postAttnNormFusedGateUp !== 'boolean') {
|
|
260
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.usePostAttnNormFusedGateUp "${String(postAttnNormFusedGateUp)}"; expected boolean.`);
|
|
261
|
+
}
|
|
262
|
+
const linearAttentionABProjectionFusion = session?.useLinearAttentionABProjectionFusion;
|
|
263
|
+
if (linearAttentionABProjectionFusion !== undefined && linearAttentionABProjectionFusion !== null && typeof linearAttentionABProjectionFusion !== 'boolean') {
|
|
264
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useLinearAttentionABProjectionFusion "${String(linearAttentionABProjectionFusion)}"; expected boolean.`);
|
|
265
|
+
}
|
|
266
|
+
const linearAttentionQKVZProjectionFusion = session?.useLinearAttentionQKVZProjectionFusion;
|
|
267
|
+
if (linearAttentionQKVZProjectionFusion !== undefined && linearAttentionQKVZProjectionFusion !== null && typeof linearAttentionQKVZProjectionFusion !== 'boolean') {
|
|
268
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useLinearAttentionQKVZProjectionFusion "${String(linearAttentionQKVZProjectionFusion)}"; expected boolean.`);
|
|
269
|
+
}
|
|
270
|
+
const linearAttentionFusedDecodeCore = session?.useLinearAttentionFusedDecodeCore;
|
|
271
|
+
if (linearAttentionFusedDecodeCore !== undefined && linearAttentionFusedDecodeCore !== null && typeof linearAttentionFusedDecodeCore !== 'boolean') {
|
|
272
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useLinearAttentionFusedDecodeCore "${String(linearAttentionFusedDecodeCore)}"; expected boolean.`);
|
|
273
|
+
}
|
|
274
|
+
const wideTileResidualFusion = session?.useWideTileResidualFusion;
|
|
275
|
+
if (wideTileResidualFusion !== undefined && wideTileResidualFusion !== null && typeof wideTileResidualFusion !== 'boolean') {
|
|
276
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileResidualFusion "${String(wideTileResidualFusion)}"; expected boolean.`);
|
|
277
|
+
}
|
|
278
|
+
const fusedRmsnormWideTile = session?.useFusedRmsnormWideTile;
|
|
279
|
+
if (fusedRmsnormWideTile !== undefined && fusedRmsnormWideTile !== null && typeof fusedRmsnormWideTile !== 'boolean') {
|
|
280
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFusedRmsnormWideTile "${String(fusedRmsnormWideTile)}"; expected boolean.`);
|
|
281
|
+
}
|
|
246
282
|
const fusedQKVSplitQKNorm = session?.useFusedQKVSplitQKNorm;
|
|
247
283
|
if (fusedQKVSplitQKNorm !== undefined && fusedQKVSplitQKNorm !== null && typeof fusedQKVSplitQKNorm !== 'boolean') {
|
|
248
284
|
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFusedQKVSplitQKNorm "${String(fusedQKVSplitQKNorm)}"; expected boolean.`);
|
|
@@ -255,17 +291,81 @@ function resolveSessionSettings(inferenceConfig, modelId) {
|
|
|
255
291
|
if (retain !== undefined && retain !== null && typeof retain !== 'boolean') {
|
|
256
292
|
throw new Error(`Manifest "${modelId}" has invalid inference.session.retainQ4KMaterialization "${String(retain)}"; expected boolean.`);
|
|
257
293
|
}
|
|
294
|
+
const lmHeadArgmaxQ4K = session?.lmHeadArgmaxQ4K;
|
|
295
|
+
if (lmHeadArgmaxQ4K !== undefined && lmHeadArgmaxQ4K !== null) {
|
|
296
|
+
if (typeof lmHeadArgmaxQ4K !== 'object' || Array.isArray(lmHeadArgmaxQ4K)) {
|
|
297
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K; expected object or null.`);
|
|
298
|
+
}
|
|
299
|
+
const fullBlockFastPath = lmHeadArgmaxQ4K.useFullBlockFastPath;
|
|
300
|
+
if (fullBlockFastPath !== undefined && typeof fullBlockFastPath !== 'boolean') {
|
|
301
|
+
throw new Error(
|
|
302
|
+
`Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K.useFullBlockFastPath ` +
|
|
303
|
+
`"${String(fullBlockFastPath)}"; expected boolean.`
|
|
304
|
+
);
|
|
305
|
+
}
|
|
306
|
+
const colsPerWorkgroup = lmHeadArgmaxQ4K.colsPerWorkgroup;
|
|
307
|
+
if (colsPerWorkgroup !== undefined && (!Number.isInteger(colsPerWorkgroup) || colsPerWorkgroup <= 0)) {
|
|
308
|
+
throw new Error(
|
|
309
|
+
`Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K.colsPerWorkgroup ` +
|
|
310
|
+
`"${String(colsPerWorkgroup)}"; expected positive integer.`
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
const threadsPerCol = lmHeadArgmaxQ4K.threadsPerCol;
|
|
314
|
+
if (threadsPerCol !== undefined && (!Number.isInteger(threadsPerCol) || threadsPerCol <= 0)) {
|
|
315
|
+
throw new Error(
|
|
316
|
+
`Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K.threadsPerCol ` +
|
|
317
|
+
`"${String(threadsPerCol)}"; expected positive integer.`
|
|
318
|
+
);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
const attentionDecodeOnline = session?.attentionDecodeOnline;
|
|
322
|
+
if (attentionDecodeOnline !== undefined && attentionDecodeOnline !== null) {
|
|
323
|
+
if (typeof attentionDecodeOnline !== 'object' || Array.isArray(attentionDecodeOnline)) {
|
|
324
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline; expected object or null.`);
|
|
325
|
+
}
|
|
326
|
+
const workgroupSize = attentionDecodeOnline.workgroupSize;
|
|
327
|
+
if (workgroupSize !== undefined && workgroupSize !== 128 && workgroupSize !== 256) {
|
|
328
|
+
throw new Error(
|
|
329
|
+
`Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline.workgroupSize ` +
|
|
330
|
+
`"${String(workgroupSize)}"; expected 128 or 256.`
|
|
331
|
+
);
|
|
332
|
+
}
|
|
333
|
+
const directKv = attentionDecodeOnline.useDirectContiguousKVLayout;
|
|
334
|
+
if (directKv !== undefined && typeof directKv !== 'boolean') {
|
|
335
|
+
throw new Error(
|
|
336
|
+
`Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline.useDirectContiguousKVLayout ` +
|
|
337
|
+
`"${String(directKv)}"; expected boolean.`
|
|
338
|
+
);
|
|
339
|
+
}
|
|
340
|
+
const outputGateFusion = attentionDecodeOnline.useOutputGateFusion;
|
|
341
|
+
if (outputGateFusion !== undefined && typeof outputGateFusion !== 'boolean') {
|
|
342
|
+
throw new Error(
|
|
343
|
+
`Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline.useOutputGateFusion ` +
|
|
344
|
+
`"${String(outputGateFusion)}"; expected boolean.`
|
|
345
|
+
);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
258
348
|
return {
|
|
259
349
|
prefillChunkSubmitMode: submit ?? null,
|
|
260
350
|
prefillTokenChunkSize: tokenChunk ?? null,
|
|
351
|
+
skipEmbeddingKVCacheWrites: skipEmbeddingKVCacheWrites ?? null,
|
|
261
352
|
useFlashPrefillAttention: flash ?? null,
|
|
353
|
+
useLargeBatchF16F32FusedGateUp: largeBatchF16F32FusedGateUp ?? null,
|
|
262
354
|
useWideTileQ4KPrefill: wide ?? null,
|
|
263
355
|
useWideTileQ4KDecode: wideDecode ?? null,
|
|
264
356
|
useSandwichRMSNormPairFusion: sandwichRmsNormPair ?? null,
|
|
265
357
|
usePostFfnNextInputRMSNormPairFusion: postFfnNextInputRmsNormPair ?? null,
|
|
358
|
+
usePostAttnNormFusedGateUp: postAttnNormFusedGateUp ?? null,
|
|
359
|
+
useLinearAttentionABProjectionFusion: linearAttentionABProjectionFusion ?? null,
|
|
360
|
+
useLinearAttentionQKVZProjectionFusion: linearAttentionQKVZProjectionFusion ?? null,
|
|
361
|
+
useLinearAttentionFusedDecodeCore: linearAttentionFusedDecodeCore ?? null,
|
|
362
|
+
useWideTileResidualFusion: wideTileResidualFusion ?? null,
|
|
363
|
+
useFusedRmsnormWideTile: fusedRmsnormWideTile ?? null,
|
|
266
364
|
useFusedQKVSplitQKNorm: fusedQKVSplitQKNorm ?? null,
|
|
267
365
|
useFusedQKVSplitQKNormRoPE: fusedQKVSplitQKNormRoPE ?? null,
|
|
268
366
|
retainQ4KMaterialization: retain ?? null,
|
|
367
|
+
lmHeadArgmaxQ4K: lmHeadArgmaxQ4K ?? null,
|
|
368
|
+
attentionDecodeOnline: attentionDecodeOnline ?? null,
|
|
269
369
|
};
|
|
270
370
|
}
|
|
271
371
|
|
|
@@ -21,6 +21,7 @@ export interface StaticExecutionPlan {
|
|
|
21
21
|
defaultMaxTokens: number;
|
|
22
22
|
readbackInterval: number | null;
|
|
23
23
|
readbackMode: 'sequential' | 'overlapped' | 'auto';
|
|
24
|
+
maxBatchDecodeTokens?: number | null;
|
|
24
25
|
ringTokens: number | null;
|
|
25
26
|
ringStop: number | null;
|
|
26
27
|
ringStaging: number | null;
|
|
@@ -53,6 +54,7 @@ export interface ExecutionSessionPlan {
|
|
|
53
54
|
maxTokens: number;
|
|
54
55
|
readbackInterval: number | null;
|
|
55
56
|
readbackMode: 'sequential' | 'overlapped' | 'auto';
|
|
57
|
+
maxBatchDecodeTokens?: number | null;
|
|
56
58
|
ringTokens: number | null;
|
|
57
59
|
ringStop: number | null;
|
|
58
60
|
ringStaging: number | null;
|