doppler-gpu 0.4.7 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +93 -144
- package/models/catalog.json +1575 -0
- package/package.json +37 -12
- package/src/cli/doppler-cli.js +28 -7
- package/src/client/doppler-registry.json +2 -2
- package/src/client/runtime/index.js +25 -2
- package/src/client/runtime/model-session.d.ts +31 -1
- package/src/client/runtime/model-session.js +31 -0
- package/src/client/runtime/model-source.d.ts +19 -2
- package/src/client/runtime/model-source.js +45 -5
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
- package/src/config/kernel-path-loader.js +1 -0
- package/src/config/kernels/kernel-ref-digests.js +26 -17
- package/src/config/kernels/registry.json +448 -27
- package/src/config/merge.js +11 -0
- package/src/config/param-validator.js +25 -0
- package/src/config/runtime/default.json +3 -1
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
- package/src/config/runtime/profiles/rerank-stability.json +43 -0
- package/src/config/runtime/profiles/rerank-throughput.json +40 -0
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
- package/src/config/schema/doppler.schema.js +40 -5
- package/src/config/schema/execution-v1.schema.d.ts +41 -0
- package/src/config/schema/execution-v1.schema.js +10 -0
- package/src/config/schema/index.d.ts +3 -1
- package/src/config/schema/speculation-self.schema.d.ts +16 -0
- package/src/config/transforms/capability-transform-resolver.js +7 -1
- package/src/config/transforms/execution-graph-transforms.js +2 -2
- package/src/debug/config.d.ts +4 -0
- package/src/experimental/orchestration/index.d.ts +10 -1
- package/src/experimental/training/suite.js +1 -1
- package/src/formats/rdrr/classification.js +3 -0
- package/src/formats/rdrr/integrity.d.ts +0 -2
- package/src/formats/rdrr/integrity.js +0 -2
- package/src/formats/rdrr/types.d.ts +1 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +75 -16
- package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
- package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
- package/src/gpu/kernels/fused_ffn.d.ts +32 -0
- package/src/gpu/kernels/fused_ffn.js +540 -19
- package/src/gpu/kernels/fused_ffn.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
- package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
- package/src/gpu/kernels/index.d.ts +21 -0
- package/src/gpu/kernels/index.js +13 -0
- package/src/gpu/kernels/linear-attention-core.js +413 -15
- package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
- package/src/gpu/kernels/lm-head-argmax.js +153 -57
- package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
- package/src/gpu/kernels/lm-head-select-logits.js +186 -0
- package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
- package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
- package/src/gpu/kernels/matmul-selection.js +5 -1
- package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
- package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
- package/src/gpu/kernels/rmsnorm-pair.js +113 -0
- package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
- package/src/gpu/kernels/rmsnorm-stats.js +250 -0
- package/src/gpu/kernels/silu.wgsl +7 -1
- package/src/gpu/kernels/silu_f16.wgsl +7 -1
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
- package/src/gpu/tensor.d.ts +1 -0
- package/src/index.d.ts +1 -1
- package/src/index.js +1 -1
- package/src/inference/browser-harness-suite-helpers.js +1 -1
- package/src/inference/browser-harness-text-helpers.js +251 -20
- package/src/inference/browser-harness.js +221 -3
- package/src/inference/kv-cache/base.js +42 -1
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
- package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
- package/src/inference/pipelines/text/attention/projections.js +174 -4
- package/src/inference/pipelines/text/attention/record.js +44 -21
- package/src/inference/pipelines/text/attention/run.js +18 -3
- package/src/inference/pipelines/text/config.d.ts +18 -0
- package/src/inference/pipelines/text/config.js +100 -0
- package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
- package/src/inference/pipelines/text/execution-plan.js +37 -0
- package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
- package/src/inference/pipelines/text/execution-v1.js +122 -8
- package/src/inference/pipelines/text/ffn/dense.js +239 -21
- package/src/inference/pipelines/text/ffn/standard.js +121 -38
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
- package/src/inference/pipelines/text/generator-helpers.js +2 -0
- package/src/inference/pipelines/text/generator-runtime.js +1 -0
- package/src/inference/pipelines/text/generator-steps.js +2 -2
- package/src/inference/pipelines/text/generator.d.ts +18 -1
- package/src/inference/pipelines/text/generator.js +297 -19
- package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
- package/src/inference/pipelines/text/init.d.ts +22 -3
- package/src/inference/pipelines/text/init.js +2 -2
- package/src/inference/pipelines/text/layer.js +87 -8
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
- package/src/inference/pipelines/text/linear-attention.js +173 -16
- package/src/inference/pipelines/text/logits/gpu.js +2 -2
- package/src/inference/pipelines/text/logits/index.js +62 -1
- package/src/inference/pipelines/text/model-load.js +6 -0
- package/src/inference/pipelines/text/ops.js +22 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
- package/src/inference/pipelines/text/probes.js +9 -4
- package/src/inference/pipelines/text/state.d.ts +4 -4
- package/src/inference/pipelines/text/state.js +4 -0
- package/src/inference/pipelines/text/types.d.ts +27 -2
- package/src/inference/pipelines/text.d.ts +21 -1
- package/src/inference/pipelines/text.js +40 -14
- package/src/inference/tokenizers/bundled.js +45 -23
- package/src/inference/tokenizers/types.d.ts +7 -4
- package/src/loader/doppler-loader.js +25 -2
- package/src/loader/index.d.ts +3 -1
- package/src/loader/index.js +2 -0
- package/src/loader/loader-types.d.ts +3 -1
- package/src/loader/manifest-config.d.ts +18 -0
- package/src/loader/manifest-config.js +11 -3
- package/src/loader/shard-cache.d.ts +2 -0
- package/src/loader/shard-cache.js +29 -1
- package/src/models/gemma4.js +8 -0
- package/src/rules/inference/capability-transforms.rules.json +145 -20
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/fused-ffn.rules.json +1 -1
- package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/rules/kernels/silu.rules.json +1 -0
- package/src/rules/rule-registry.js +2 -0
- package/src/storage/artifact-storage-context.js +36 -0
- package/src/tooling/node-browser-command-runner.js +42 -7
- package/src/tooling/opfs-cache.d.ts +21 -1
- package/src/tooling/opfs-cache.js +168 -32
- package/src/tooling-exports/storage.d.ts +1 -1
- package/src/tooling-exports/storage.js +1 -1
- package/src/types/gpu.d.ts +25 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +1593 -0
- package/tests/kernels/browser/test-page.js +2629 -0
- package/tests/kernels/harness/benchmark.js +225 -0
- package/tests/kernels/harness/buffer-utils.js +88 -0
- package/tests/kernels/harness/tolerance.js +174 -0
- package/tests/kernels/reference/adam.js +15 -0
- package/tests/kernels/reference/attention.js +158 -0
- package/tests/kernels/reference/backward.js +96 -0
- package/tests/kernels/reference/bias-add.js +20 -0
- package/tests/kernels/reference/conv2d.js +58 -0
- package/tests/kernels/reference/cross-entropy.js +26 -0
- package/tests/kernels/reference/dequant.js +312 -0
- package/tests/kernels/reference/embed.js +15 -0
- package/tests/kernels/reference/gather.js +61 -0
- package/tests/kernels/reference/gelu.js +37 -0
- package/tests/kernels/reference/groupnorm.js +36 -0
- package/tests/kernels/reference/index.js +89 -0
- package/tests/kernels/reference/layernorm.js +22 -0
- package/tests/kernels/reference/matmul.js +57 -0
- package/tests/kernels/reference/modulate.js +24 -0
- package/tests/kernels/reference/moe-gather.js +83 -0
- package/tests/kernels/reference/pixel-shuffle.js +20 -0
- package/tests/kernels/reference/residual.js +55 -0
- package/tests/kernels/reference/rmsnorm.js +52 -0
- package/tests/kernels/reference/rope.js +79 -0
- package/tests/kernels/reference/sample.js +97 -0
- package/tests/kernels/reference/scatter-add.js +58 -0
- package/tests/kernels/reference/silu.js +52 -0
- package/tests/kernels/reference/softmax.js +86 -0
- package/tests/kernels/reference/split-qkv.js +58 -0
- package/tests/kernels/reference/topk.js +88 -0
- package/tests/kernels/reference/upsample2d.js +37 -0
- package/BRANDING.md +0 -14
- package/SECURITY.md +0 -19
- package/assets/doppler-runtime-map.svg +0 -82
- package/src/bootstrap.js +0 -30
- package/src/cli/doppler-cli.d.ts +0 -46
- package/src/cli/doppler-quickstart.d.ts +0 -45
- package/src/cli/doppler-serve.d.ts +0 -210
- package/src/cli/runtime-profiles.d.ts +0 -40
- package/src/client/doppler-api.browser.d.ts +0 -1
- package/src/client/doppler-api.browser.js +0 -25
- package/src/client/doppler-registry.d.ts +0 -54
- package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
- package/src/client/runtime/source-runtime.d.ts +0 -26
- package/src/client/runtime/storage.d.ts +0 -5
- package/src/config/README.md +0 -61
- package/src/config/conversion/README.md +0 -62
- package/src/config/goal-completion-matrix.json +0 -552
- package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
- package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
- package/src/config/kernels/moe/mixtral.paths.json +0 -46
- package/src/config/kernels/registry.d.ts +0 -51
- package/src/config/merge-contract-check.d.ts +0 -16
- package/src/config/merge-contract-check.js +0 -319
- package/src/config/merge-helpers.d.ts +0 -60
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
- package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
- package/src/config/param-categories.d.ts +0 -8
- package/src/config/param-validator.d.ts +0 -33
- package/src/config/platforms/loader.d.ts +0 -38
- package/src/config/quantization-contract-check.d.ts +0 -12
- package/src/config/quantization-contract-check.js +0 -91
- package/src/config/runtime/README.md +0 -28
- package/src/config/runtime-merge.d.ts +0 -5
- package/src/config/schema/claim-evidence-contract.schema.json +0 -93
- package/src/config/schema/command-surface-contract.schema.json +0 -81
- package/src/config/schema/goal-completion-matrix.schema.json +0 -223
- package/src/config/schema/model-artifact-contract.schema.json +0 -119
- package/src/config/schema/platform.schema.js +0 -1
- package/src/config/schema/policy-schema-registry.json +0 -31
- package/src/config/schema/subsystem-support-contract.schema.json +0 -48
- package/src/config/support-tiers/subsystems.json +0 -502
- package/src/config/supported-operations.d.ts +0 -14
- package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
- package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
- package/src/converter/conversion-plan.d.ts +0 -56
- package/src/converter/index.d.ts +0 -98
- package/src/converter/index.js +0 -59
- package/src/converter/parsers/diffusion.d.ts +0 -50
- package/src/converter/parsers/gguf.d.ts +0 -31
- package/src/converter/parsers/index.d.ts +0 -21
- package/src/converter/parsers/index.js +0 -12
- package/src/converter/parsers/transformer.d.ts +0 -31
- package/src/converter/quantization-info.d.ts +0 -38
- package/src/converter/rope-config.d.ts +0 -16
- package/src/converter/tokenizer-utils.d.ts +0 -12
- package/src/debug/reference/README.md +0 -114
- package/src/debug/reference/hf_attn_debug.py +0 -114
- package/src/debug/reference/hf_embed_check.py +0 -89
- package/src/debug/reference/hf_layer_out.py +0 -100
- package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
- package/src/debug/reference/hf_rope_check.py +0 -116
- package/src/debug/reference/hf_weights.py +0 -75
- package/src/errors/doppler-error.d.ts +0 -20
- package/src/experimental/bridge/extension/background.d.ts +0 -14
- package/src/experimental/bridge/extension/background.js +0 -168
- package/src/experimental/bridge/extension/manifest.json +0 -34
- package/src/experimental/bridge/extension-client.d.ts +0 -114
- package/src/experimental/bridge/index.d.ts +0 -69
- package/src/experimental/bridge/node-bridge.js +0 -24
- package/src/experimental/bridge/protocol.d.ts +0 -96
- package/src/experimental/browser/gguf-importer.d.ts +0 -136
- package/src/experimental/browser/gguf-importer.js +0 -532
- package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
- package/src/experimental/browser/quantization.d.ts +0 -74
- package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
- package/src/experimental/browser/shard-io-browser.d.ts +0 -57
- package/src/experimental/browser/shard-io-browser.js +0 -89
- package/src/experimental/browser/tensor-source-download.d.ts +0 -27
- package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
- package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
- package/src/experimental/distribution/p2p-transport.js +0 -75
- package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
- package/src/experimental/hotswap/manifest.d.ts +0 -42
- package/src/experimental/hotswap/model-swap.js +0 -42
- package/src/experimental/hotswap/runtime.d.ts +0 -31
- package/src/experimental/training/README.md +0 -164
- package/src/experimental/training/artifacts.d.ts +0 -160
- package/src/experimental/training/attention-backward.d.ts +0 -30
- package/src/experimental/training/attention-forward.d.ts +0 -22
- package/src/experimental/training/attention-forward.js +0 -82
- package/src/experimental/training/autograd.d.ts +0 -51
- package/src/experimental/training/backward-kernels.js +0 -39
- package/src/experimental/training/checkpoint-watch.d.ts +0 -8
- package/src/experimental/training/checkpoint.d.ts +0 -36
- package/src/experimental/training/clip.d.ts +0 -9
- package/src/experimental/training/dataloader.d.ts +0 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
- package/src/experimental/training/datasets/gepa-frontier.js +0 -220
- package/src/experimental/training/datasets/index.d.ts +0 -24
- package/src/experimental/training/datasets/index.js +0 -13
- package/src/experimental/training/datasets/jsonl.d.ts +0 -11
- package/src/experimental/training/datasets/reploid.d.ts +0 -3
- package/src/experimental/training/datasets/reploid.js +0 -36
- package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
- package/src/experimental/training/datasets/teacher-traces.js +0 -279
- package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
- package/src/experimental/training/datasets/token-batch.d.ts +0 -21
- package/src/experimental/training/datasets/token-batch.js +0 -52
- package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
- package/src/experimental/training/datasets/translation-pairs.js +0 -49
- package/src/experimental/training/distillation/artifacts.d.ts +0 -71
- package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
- package/src/experimental/training/distillation/dataset.d.ts +0 -59
- package/src/experimental/training/distillation/eval.d.ts +0 -34
- package/src/experimental/training/distillation/index.d.ts +0 -29
- package/src/experimental/training/distillation/runtime.d.ts +0 -20
- package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
- package/src/experimental/training/distillation/stage-a.d.ts +0 -45
- package/src/experimental/training/distillation/stage-b.d.ts +0 -24
- package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
- package/src/experimental/training/distillation/suite-data.d.ts +0 -45
- package/src/experimental/training/export.d.ts +0 -41
- package/src/experimental/training/index.d.ts +0 -74
- package/src/experimental/training/index.js +0 -63
- package/src/experimental/training/lora-pipeline.d.ts +0 -183
- package/src/experimental/training/lora.d.ts +0 -19
- package/src/experimental/training/loss-scaling.d.ts +0 -21
- package/src/experimental/training/loss.d.ts +0 -10
- package/src/experimental/training/objectives/base.d.ts +0 -58
- package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
- package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
- package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
- package/src/experimental/training/objectives/index.d.ts +0 -12
- package/src/experimental/training/objectives/index.js +0 -6
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
- package/src/experimental/training/operator-agent-eval.d.ts +0 -65
- package/src/experimental/training/operator-artifacts.d.ts +0 -62
- package/src/experimental/training/operator-command.d.ts +0 -5
- package/src/experimental/training/operator-eval.d.ts +0 -48
- package/src/experimental/training/operator-scoreboard.d.ts +0 -5
- package/src/experimental/training/optimizer.d.ts +0 -22
- package/src/experimental/training/runner.d.ts +0 -249
- package/src/experimental/training/suite.d.ts +0 -299
- package/src/experimental/training/tensor-factory.d.ts +0 -9
- package/src/experimental/training/trainer.d.ts +0 -89
- package/src/experimental/training/ul_dataset.d.ts +0 -47
- package/src/experimental/training/ul_schedule.d.ts +0 -6
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
- package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
- package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
- package/src/experimental/training/workload-packs/registry.json +0 -278
- package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
- package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
- package/src/experimental/training/workloads.d.ts +0 -201
- package/src/formats/gguf/types.d.ts +0 -74
- package/src/formats/index.d.ts +0 -75
- package/src/formats/index.js +0 -19
- package/src/formats/litert/types.d.ts +0 -74
- package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
- package/src/formats/rdrr/distributed/validation.d.ts +0 -23
- package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
- package/src/formats/safetensors/types.d.ts +0 -67
- package/src/formats/tflite/types.d.ts +0 -61
- package/src/formats/tokenizer/index.d.ts +0 -5
- package/src/formats/tokenizer/index.js +0 -3
- package/src/formats/tokenizer/types.d.ts +0 -9
- package/src/gpu/kernel-runtime.d.ts +0 -13
- package/src/gpu/kernel-selection-cache.d.ts +0 -13
- package/src/gpu/kernel-selection-log.d.ts +0 -4
- package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
- package/src/gpu/kernel-tuner/cache.d.ts +0 -55
- package/src/gpu/kernel-tuner/index.d.ts +0 -59
- package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
- package/src/gpu/kernel-tuner/types.d.ts +0 -101
- package/src/gpu/kernel-tuner/types.js +0 -4
- package/src/gpu/kernel-tuner.d.ts +0 -33
- package/src/gpu/kernels/README.md +0 -166
- package/src/gpu/kernels/check-finiteness.d.ts +0 -15
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
- package/src/gpu/kernels/check-stop.d.ts +0 -31
- package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
- package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
- package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
- package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
- package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
- package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
- package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
- package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
- package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
- package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
- package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
- package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
- package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
- package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
- package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
- package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
- package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
- package/src/gpu/kernels/constants.d.ts +0 -129
- package/src/gpu/kernels/dispatch.d.ts +0 -72
- package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
- package/src/gpu/kernels/kernel-base.d.ts +0 -33
- package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
- package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
- package/src/gpu/kernels/matmul-selection.d.ts +0 -87
- package/src/gpu/kernels/rep-penalty.d.ts +0 -21
- package/src/gpu/kernels/rule-matcher.d.ts +0 -30
- package/src/gpu/kernels/rule-registry.d.ts +0 -7
- package/src/gpu/kernels/types.js +0 -4
- package/src/gpu/perf-guards.d.ts +0 -13
- package/src/gpu/profiler.d.ts +0 -93
- package/src/gpu/readback-utils.d.ts +0 -16
- package/src/gpu/readback-utils.js +0 -49
- package/src/gpu/submit-probe.d.ts +0 -12
- package/src/index-browser.d.ts +0 -2
- package/src/index-browser.js +0 -2
- package/src/inference/README.md +0 -593
- package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
- package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
- package/src/inference/browser-harness-model-helpers.d.ts +0 -10
- package/src/inference/browser-harness-report-helpers.d.ts +0 -7
- package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
- package/src/inference/browser-harness-text-helpers.d.ts +0 -152
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
- package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
- package/src/inference/network-evolution.js +0 -79
- package/src/inference/pipelines/context.d.ts +0 -21
- package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
- package/src/inference/pipelines/diffusion/index.d.ts +0 -3
- package/src/inference/pipelines/diffusion/index.js +0 -3
- package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
- package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
- package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
- package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
- package/src/inference/pipelines/diffusion/types.js +0 -1
- package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
- package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
- package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
- package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
- package/src/inference/pipelines/dream/pipeline.js +0 -8
- package/src/inference/pipelines/energy/index.d.ts +0 -1
- package/src/inference/pipelines/energy/index.js +0 -1
- package/src/inference/pipelines/energy/types.js +0 -1
- package/src/inference/pipelines/energy-head/index.d.ts +0 -6
- package/src/inference/pipelines/energy-head/index.js +0 -6
- package/src/inference/pipelines/factory.d.ts +0 -10
- package/src/inference/pipelines/index.d.ts +0 -22
- package/src/inference/pipelines/index.js +0 -19
- package/src/inference/pipelines/registry.d.ts +0 -15
- package/src/inference/pipelines/rng.d.ts +0 -2
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
- package/src/inference/pipelines/text/attention/index.d.ts +0 -24
- package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
- package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
- package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
- package/src/inference/pipelines/text/attention/record.d.ts +0 -36
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
- package/src/inference/pipelines/text/attention/run.d.ts +0 -38
- package/src/inference/pipelines/text/attention/types.d.ts +0 -125
- package/src/inference/pipelines/text/attention.d.ts +0 -23
- package/src/inference/pipelines/text/attention.js +0 -12
- package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
- package/src/inference/pipelines/text/buffer-types.js +0 -4
- package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
- package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
- package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
- package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
- package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
- package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
- package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
- package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
- package/src/inference/pipelines/text/embed.d.ts +0 -69
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
- package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
- package/src/inference/pipelines/text/failure-reduction.js +0 -180
- package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
- package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
- package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
- package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
- package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
- package/src/inference/pipelines/text/ffn.d.ts +0 -31
- package/src/inference/pipelines/text/ffn.js +0 -18
- package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
- package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
- package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
- package/src/inference/pipelines/text/index.d.ts +0 -5
- package/src/inference/pipelines/text/index.js +0 -6
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
- package/src/inference/pipelines/text/layer.d.ts +0 -118
- package/src/inference/pipelines/text/load-errors.d.ts +0 -21
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
- package/src/inference/pipelines/text/logits/types.js +0 -4
- package/src/inference/pipelines/text/logits.d.ts +0 -27
- package/src/inference/pipelines/text/logits.js +0 -16
- package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
- package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
- package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
- package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
- package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
- package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
- package/src/inference/pipelines/text/operator-events.d.ts +0 -125
- package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
- package/src/inference/pipelines/text/ops.d.ts +0 -222
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
- package/src/inference/pipelines/text/probes.d.ts +0 -28
- package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
- package/src/inference/pipelines/text/stage-names.d.ts +0 -88
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
- package/src/inference/pipelines/text/types.js +0 -4
- package/src/inference/tokenizers/base.d.ts +0 -44
- package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
- package/src/inference/tokenizers/bpe.d.ts +0 -27
- package/src/inference/tokenizers/bundled.d.ts +0 -67
- package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
- package/src/inference/tokenizers/types.js +0 -7
- package/src/loader/embedding-limit-preflight.d.ts +0 -31
- package/src/loader/per-layer-input-loader.d.ts +0 -30
- package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
- package/src/loader/tensors/source-transform.d.ts +0 -65
- package/src/memory/address-table.d.ts +0 -96
- package/src/rules/execution-rules-contract-check.d.ts +0 -21
- package/src/rules/layer-pattern-contract-check.d.ts +0 -17
- package/src/rules/rule-registry.d.ts +0 -77
- package/src/storage/artifact-storage-context.d.ts +0 -63
- package/src/storage/backends/idb-store.d.ts +0 -52
- package/src/storage/backends/memory-store.d.ts +0 -36
- package/src/storage/backends/opfs-store.d.ts +0 -41
- package/src/storage/blake3.d.ts +0 -17
- package/src/storage/download-types.d.ts +0 -157
- package/src/storage/downloader.d.ts +0 -103
- package/src/storage/index.d.ts +0 -274
- package/src/storage/index.js +0 -201
- package/src/storage/preflight.d.ts +0 -144
- package/src/storage/preflight.js +0 -360
- package/src/storage/quickstart-downloader.d.ts +0 -157
- package/src/storage/quickstart-downloader.js +0 -289
- package/src/storage/source-artifact-store.d.ts +0 -59
- package/src/sw.js +0 -184
- package/src/tooling/command-api-constants.d.ts +0 -9
- package/src/tooling/command-api-family-normalizers.d.ts +0 -10
- package/src/tooling/command-api-helpers.d.ts +0 -23
- package/src/tooling/command-runner-shared.d.ts +0 -89
- package/src/tooling/conversion-config-materializer.d.ts +0 -14
- package/src/tooling/conversion-config-materializer.js +0 -96
- package/src/tooling/diagnose-runner.d.ts +0 -28
- package/src/tooling/hf-registry-utils.d.ts +0 -55
- package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
- package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
- package/src/tooling/kernel-path-builder/index.d.ts +0 -65
- package/src/tooling/kernel-path-builder/index.js +0 -1361
- package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
- package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
- package/src/tooling/lean-execution-contract-runner.js +0 -158
- package/src/tooling/lean-execution-contract.d.ts +0 -16
- package/src/tooling/lean-execution-contract.js +0 -81
- package/src/tooling/litert-package-runtime.d.ts +0 -46
- package/src/tooling/node-convert-worker-pool.d.ts +0 -16
- package/src/tooling/node-convert-worker.d.ts +0 -1
- package/src/tooling/node-converter.d.ts +0 -1
- package/src/tooling/node-file-fetch.d.ts +0 -1
- package/src/tooling/node-source-runtime.d.ts +0 -24
- package/src/tooling/node-webgpu.d.ts +0 -17
- package/src/tooling/precision-replay-math.js +0 -174
- package/src/tooling/program-bundle-host.d.ts +0 -16
- package/src/tooling/program-bundle-host.js +0 -20
- package/src/tooling/runtime-input-composition.d.ts +0 -37
- package/src/tooling/source-package-profiles.d.ts +0 -50
- package/src/tooling/source-runtime-converter-config.d.ts +0 -19
- package/src/tooling/source-runtime-materializer.d.ts +0 -6
- package/src/tooling/source-runtime-materializer.js +0 -89
- package/src/tooling-exports.browser.d.ts +0 -7
- package/src/types/chrome.d.ts +0 -36
- package/src/types/chrome.js +0 -1
- package/src/types/index.d.ts +0 -1
- package/src/types/index.js +0 -1
- package/src/utils/array-buffer.d.ts +0 -4
- package/src/utils/canonical-hash.d.ts +0 -7
- package/src/utils/clone-json.d.ts +0 -1
- package/src/utils/hf-resolve-url.d.ts +0 -16
- package/src/utils/index.d.ts +0 -10
- package/src/utils/index.js +0 -10
- package/src/utils/load-json.d.ts +0 -5
- package/src/utils/plain-object.d.ts +0 -1
- package/src/utils/sha256.d.ts +0 -5
- package/src/utils/stable-sort-object.d.ts +0 -1
|
@@ -29,6 +29,13 @@ function assertOptionalPositiveInt(value, label) {
|
|
|
29
29
|
return value;
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
+
function assertOptionalNullablePositiveInt(value, label) {
|
|
33
|
+
if (value === undefined || value === null) {
|
|
34
|
+
return value;
|
|
35
|
+
}
|
|
36
|
+
return assertOptionalPositiveInt(value, label);
|
|
37
|
+
}
|
|
38
|
+
|
|
32
39
|
function assertRequiredPositiveInt(value, label) {
|
|
33
40
|
if (!Number.isInteger(value) || value < 1) {
|
|
34
41
|
throw new Error(`[ExecutionPlan] ${label} must be a positive integer; got ${JSON.stringify(value)}.`);
|
|
@@ -139,6 +146,7 @@ function createStaticExecutionPlan({
|
|
|
139
146
|
defaultMaxTokens: batchingConfig.maxTokens,
|
|
140
147
|
readbackInterval: batchingConfig.readbackInterval,
|
|
141
148
|
readbackMode: batchingConfig.readbackMode,
|
|
149
|
+
maxBatchDecodeTokens: batchingConfig.maxBatchDecodeTokens,
|
|
142
150
|
ringTokens: batchingConfig.ringTokens,
|
|
143
151
|
ringStop: batchingConfig.ringStop,
|
|
144
152
|
ringStaging: batchingConfig.ringStaging,
|
|
@@ -224,6 +232,10 @@ export function compileExecutionPlanState(options) {
|
|
|
224
232
|
maxTokens: assertRequiredPositiveInt(generationConfig.maxTokens, 'runtime.inference.generation.maxTokens'),
|
|
225
233
|
readbackInterval: decodeLoopConfig.readbackInterval,
|
|
226
234
|
readbackMode: assertReadbackMode(decodeLoopConfig.readbackMode),
|
|
235
|
+
maxBatchDecodeTokens: assertOptionalNullablePositiveInt(
|
|
236
|
+
decodeLoopConfig.maxBatchDecodeTokens,
|
|
237
|
+
'runtimeConfig.inference.session.decodeLoop.maxBatchDecodeTokens'
|
|
238
|
+
),
|
|
227
239
|
ringTokens: decodeLoopConfig.ringTokens,
|
|
228
240
|
ringStop: decodeLoopConfig.ringStop,
|
|
229
241
|
ringStaging: decodeLoopConfig.ringStaging,
|
|
@@ -406,6 +418,7 @@ export function resolveExecutionSessionPlan(container, options = {}) {
|
|
|
406
418
|
maxTokens: overrides.maxTokens ?? activePlan.defaultMaxTokens,
|
|
407
419
|
readbackInterval: overrides.readbackInterval ?? activePlan.readbackInterval,
|
|
408
420
|
readbackMode: activePlan.readbackMode,
|
|
421
|
+
maxBatchDecodeTokens: activePlan.maxBatchDecodeTokens,
|
|
409
422
|
ringTokens: overrides.ringTokens ?? activePlan.ringTokens,
|
|
410
423
|
ringStop: overrides.ringStop ?? activePlan.ringStop,
|
|
411
424
|
ringStaging: overrides.ringStaging ?? activePlan.ringStaging,
|
|
@@ -434,6 +447,18 @@ export function isBatchDecodeEnabled(config) {
|
|
|
434
447
|
}
|
|
435
448
|
|
|
436
449
|
export function resolveMaxBatchDecodeTokens(config) {
|
|
450
|
+
if (config.configuredMaxBatchDecodeTokens !== undefined) {
|
|
451
|
+
const value = config.configuredMaxBatchDecodeTokens;
|
|
452
|
+
if (value === null) {
|
|
453
|
+
return null;
|
|
454
|
+
}
|
|
455
|
+
if (!Number.isInteger(value) || value < 1) {
|
|
456
|
+
throw new Error(
|
|
457
|
+
`[ExecutionPlan] configured maxBatchDecodeTokens must be null or a positive integer; got ${JSON.stringify(value)}.`
|
|
458
|
+
);
|
|
459
|
+
}
|
|
460
|
+
return value;
|
|
461
|
+
}
|
|
437
462
|
const value = selectRuleValue('inference', 'execution', 'maxBatchDecodeTokens', {
|
|
438
463
|
hasHotVocabularyBatchDecode: config.hasHotVocabularyBatchDecode === true,
|
|
439
464
|
hasGpuSplitPerLayerInputs: config.hasGpuSplitPerLayerInputs === true,
|
|
@@ -457,6 +482,18 @@ export function resolveMaxBatchDecodeTokens(config) {
|
|
|
457
482
|
}
|
|
458
483
|
|
|
459
484
|
export function resolvePrefillRecorderChunkLayers(config) {
|
|
485
|
+
if (config.configuredPrefillChunkLayers !== undefined) {
|
|
486
|
+
const configured = config.configuredPrefillChunkLayers;
|
|
487
|
+
if (configured === null) {
|
|
488
|
+
throw new Error('[ExecutionPlan] configured prefillChunkLayers cannot be null.');
|
|
489
|
+
}
|
|
490
|
+
if (!Number.isInteger(configured) || configured < 1) {
|
|
491
|
+
throw new Error(
|
|
492
|
+
`[ExecutionPlan] configured prefillChunkLayers must be a positive integer; got ${JSON.stringify(configured)}.`
|
|
493
|
+
);
|
|
494
|
+
}
|
|
495
|
+
return configured;
|
|
496
|
+
}
|
|
460
497
|
const value = selectRuleValue('inference', 'execution', 'prefillRecorderChunkLayers', {
|
|
461
498
|
hasGpuSplitPerLayerInputs: config.hasGpuSplitPerLayerInputs === true,
|
|
462
499
|
numTokens: config.numTokens,
|
|
@@ -21,6 +21,12 @@ export interface ExecutionV1LaneIntegrity {
|
|
|
21
21
|
};
|
|
22
22
|
status: 'matches' | 'transformed';
|
|
23
23
|
transforms: string[];
|
|
24
|
+
policy?: {
|
|
25
|
+
kind: string | null;
|
|
26
|
+
dtypeEffect: string | null;
|
|
27
|
+
reason: string | null;
|
|
28
|
+
evidence: string[];
|
|
29
|
+
} | null;
|
|
24
30
|
}
|
|
25
31
|
|
|
26
32
|
export interface ExecutionV1CompiledState {
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
} from './execution-runtime-builders.js';
|
|
10
10
|
import { mergeKernelPathPolicy } from '../../../config/merge-helpers.js';
|
|
11
11
|
import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
|
|
12
|
+
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
12
13
|
import { buildOpIdFromExecutionStep } from './operator-identity.js';
|
|
13
14
|
import {
|
|
14
15
|
resolveCapabilityTransforms,
|
|
@@ -32,6 +33,56 @@ function isSessionCapabilityTransform(name) {
|
|
|
32
33
|
return SESSION_CAPABILITY_TRANSFORMS.has(name);
|
|
33
34
|
}
|
|
34
35
|
|
|
36
|
+
function resolveExecutionCapabilities(capabilities) {
|
|
37
|
+
if (capabilities && typeof capabilities === 'object') {
|
|
38
|
+
return capabilities;
|
|
39
|
+
}
|
|
40
|
+
return {
|
|
41
|
+
hasSubgroups: false,
|
|
42
|
+
hasSubgroupsF16: false,
|
|
43
|
+
hasF16: false,
|
|
44
|
+
hasTimestampQuery: false,
|
|
45
|
+
maxBufferSize: 0,
|
|
46
|
+
maxWorkgroupSize: 0,
|
|
47
|
+
maxWorkgroupStorageSize: 0,
|
|
48
|
+
adapterInfo: {
|
|
49
|
+
vendor: 'unknown',
|
|
50
|
+
architecture: 'unknown',
|
|
51
|
+
device: 'unknown',
|
|
52
|
+
description: 'execution-v1-no-f16-proof',
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function resolveExecutionSessionKVDtype(session, manifestInference, capabilities, useGPU) {
|
|
58
|
+
const runtimeKV = session?.kvcache ?? null;
|
|
59
|
+
if (!runtimeKV) {
|
|
60
|
+
return session;
|
|
61
|
+
}
|
|
62
|
+
if (useGPU == null && !capabilities) {
|
|
63
|
+
return session;
|
|
64
|
+
}
|
|
65
|
+
const attnSoftcap = manifestInference?.attention?.attnLogitSoftcapping;
|
|
66
|
+
const forceF32Softcap = runtimeKV.forceF32Softcap === true;
|
|
67
|
+
const forceF32KV = attnSoftcap != null && attnSoftcap > 0 && forceF32Softcap;
|
|
68
|
+
const kvDtype = selectRuleValue('inference', 'dtype', 'kvCacheDtype', {
|
|
69
|
+
requested: runtimeKV.kvDtype,
|
|
70
|
+
useGPU: useGPU === true,
|
|
71
|
+
hasF16: capabilities?.hasF16 === true,
|
|
72
|
+
forceF32: forceF32KV,
|
|
73
|
+
});
|
|
74
|
+
if (kvDtype === runtimeKV.kvDtype) {
|
|
75
|
+
return session;
|
|
76
|
+
}
|
|
77
|
+
return {
|
|
78
|
+
...session,
|
|
79
|
+
kvcache: {
|
|
80
|
+
...runtimeKV,
|
|
81
|
+
kvDtype,
|
|
82
|
+
},
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
35
86
|
function mergeExecutionV1Session(manifestSession, runtimeSession) {
|
|
36
87
|
return mergeRuntimeValues(
|
|
37
88
|
manifestSession ?? {},
|
|
@@ -297,6 +348,34 @@ function preserveRuntimeDecodeLoop(updatedInference, runtimeConfig) {
|
|
|
297
348
|
};
|
|
298
349
|
}
|
|
299
350
|
|
|
351
|
+
function preserveConfiguredKernelPath(updatedInference, runtimeConfig) {
|
|
352
|
+
const configuredInference = runtimeConfig?.inference;
|
|
353
|
+
const configuredKernelPath = configuredInference?.kernelPath;
|
|
354
|
+
if (configuredKernelPath == null) {
|
|
355
|
+
return updatedInference;
|
|
356
|
+
}
|
|
357
|
+
const configuredSession = configuredInference?.session;
|
|
358
|
+
const hasConfiguredSessionCompute = hasOwnProperty(configuredSession, 'compute');
|
|
359
|
+
const hasConfiguredSessionKVCache = hasOwnProperty(configuredSession, 'kvcache');
|
|
360
|
+
return {
|
|
361
|
+
...updatedInference,
|
|
362
|
+
kernelPath: configuredKernelPath,
|
|
363
|
+
kernelPathSource: 'config',
|
|
364
|
+
...(hasOwnProperty(configuredInference, 'compute')
|
|
365
|
+
? { compute: configuredInference.compute }
|
|
366
|
+
: {}),
|
|
367
|
+
...(hasConfiguredSessionCompute || hasConfiguredSessionKVCache
|
|
368
|
+
? {
|
|
369
|
+
session: {
|
|
370
|
+
...updatedInference.session,
|
|
371
|
+
...(hasConfiguredSessionCompute ? { compute: configuredSession.compute } : {}),
|
|
372
|
+
...(hasConfiguredSessionKVCache ? { kvcache: configuredSession.kvcache } : {}),
|
|
373
|
+
},
|
|
374
|
+
}
|
|
375
|
+
: {}),
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
|
|
300
379
|
const EXECUTION_V1_PROJECTION_OPS = new Set([
|
|
301
380
|
'q_proj', 'k_proj', 'v_proj', 'o_proj',
|
|
302
381
|
'gate_proj', 'up_proj', 'down_proj',
|
|
@@ -651,7 +730,16 @@ export function compileExecutionV1(options = {}) {
|
|
|
651
730
|
const modelId = options.modelId ?? 'model';
|
|
652
731
|
const numLayers = options.numLayers ?? 0;
|
|
653
732
|
const headDim = Number.isFinite(options.headDim) ? Math.floor(options.headDim) : null;
|
|
654
|
-
const
|
|
733
|
+
const hasCapabilityProof = options.capabilities && typeof options.capabilities === 'object';
|
|
734
|
+
const hasUseGPUOption = hasOwnProperty(options, 'useGPU');
|
|
735
|
+
const useGPU = hasUseGPUOption
|
|
736
|
+
? options.useGPU === true
|
|
737
|
+
: hasCapabilityProof
|
|
738
|
+
? true
|
|
739
|
+
: null;
|
|
740
|
+
const capabilities = hasCapabilityProof || hasUseGPUOption
|
|
741
|
+
? resolveExecutionCapabilities(options.capabilities ?? null)
|
|
742
|
+
: null;
|
|
655
743
|
const platform = options.platform ?? null;
|
|
656
744
|
const runtimeSession = options.runtimeSession ?? null;
|
|
657
745
|
const runtimeCompute = options.runtimeCompute ?? null;
|
|
@@ -669,7 +757,13 @@ export function compileExecutionV1(options = {}) {
|
|
|
669
757
|
manifestInference.session ?? {},
|
|
670
758
|
runtimeSession ?? {}
|
|
671
759
|
);
|
|
672
|
-
const
|
|
760
|
+
const declaredSession = mergedSession;
|
|
761
|
+
const session = resolveExecutionSessionKVDtype(
|
|
762
|
+
mergedSession,
|
|
763
|
+
manifestInference,
|
|
764
|
+
capabilities,
|
|
765
|
+
useGPU
|
|
766
|
+
);
|
|
673
767
|
|
|
674
768
|
if (!session?.compute?.defaults?.activationDtype) {
|
|
675
769
|
throw new Error('[ExecutionV1] session.compute.defaults.activationDtype is required.');
|
|
@@ -678,7 +772,12 @@ export function compileExecutionV1(options = {}) {
|
|
|
678
772
|
const activationDtype = session.compute.defaults.activationDtype;
|
|
679
773
|
const mathDtype = session.compute.defaults.mathDtype ?? null;
|
|
680
774
|
const accumDtype = session.compute.defaults.accumDtype ?? null;
|
|
775
|
+
const requestedActivationDtype = runtimeCompute?.activationDtype ?? activationDtype;
|
|
681
776
|
const kvDtype = requireSessionKVDtype(session);
|
|
777
|
+
const declaredActivationDtype = declaredSession.compute?.defaults?.activationDtype ?? activationDtype;
|
|
778
|
+
const declaredMathDtype = declaredSession.compute?.defaults?.mathDtype ?? mathDtype;
|
|
779
|
+
const declaredAccumDtype = declaredSession.compute?.defaults?.accumDtype ?? accumDtype;
|
|
780
|
+
const declaredKvDtype = declaredSession.kvcache?.kvDtype ?? kvDtype;
|
|
682
781
|
const layerTypes = manifestInference?.layerPattern?.layerTypes ?? null;
|
|
683
782
|
|
|
684
783
|
if (execution !== manifestExecution) {
|
|
@@ -690,7 +789,9 @@ export function compileExecutionV1(options = {}) {
|
|
|
690
789
|
// -------------------------------------------------------------------------
|
|
691
790
|
let appliedTransformNames = [];
|
|
692
791
|
let graphWasTransformed = false;
|
|
792
|
+
let capabilityTransformPolicy = null;
|
|
693
793
|
const graphContext = {
|
|
794
|
+
requestedActivationDtype,
|
|
694
795
|
activationDtype,
|
|
695
796
|
mathDtype,
|
|
696
797
|
accumDtype,
|
|
@@ -704,6 +805,12 @@ export function compileExecutionV1(options = {}) {
|
|
|
704
805
|
|
|
705
806
|
if (capabilities) {
|
|
706
807
|
const resolved = resolveCapabilityTransforms(capabilities, platform, graphContext);
|
|
808
|
+
capabilityTransformPolicy = {
|
|
809
|
+
kind: resolved.kind ?? null,
|
|
810
|
+
dtypeEffect: resolved.dtypeEffect ?? null,
|
|
811
|
+
reason: resolved.reason ?? null,
|
|
812
|
+
evidence: Array.isArray(resolved.evidence) ? [...resolved.evidence] : [],
|
|
813
|
+
};
|
|
707
814
|
const sourceScope = kernelPathPolicy.sourceScope ?? kernelPathPolicy.allowSources ?? [];
|
|
708
815
|
const remapAllowed = kernelPathPolicy.mode === 'capability-aware'
|
|
709
816
|
&& kernelPathPolicy.onIncompatible === 'remap'
|
|
@@ -845,10 +952,10 @@ export function compileExecutionV1(options = {}) {
|
|
|
845
952
|
// any drift here is owned by capability transforms (e.g., widenToF32Activations
|
|
846
953
|
// on hasF16=false GPUs).
|
|
847
954
|
const declaredLane = {
|
|
848
|
-
activationDtype,
|
|
849
|
-
mathDtype,
|
|
850
|
-
accumDtype,
|
|
851
|
-
kvDtype,
|
|
955
|
+
activationDtype: declaredActivationDtype,
|
|
956
|
+
mathDtype: declaredMathDtype,
|
|
957
|
+
accumDtype: declaredAccumDtype,
|
|
958
|
+
kvDtype: declaredKvDtype,
|
|
852
959
|
};
|
|
853
960
|
const executedLane = {
|
|
854
961
|
activationDtype: effectiveSession.compute?.defaults?.activationDtype ?? null,
|
|
@@ -867,6 +974,7 @@ export function compileExecutionV1(options = {}) {
|
|
|
867
974
|
executed: executedLane,
|
|
868
975
|
status: laneFieldDelta ? 'transformed' : 'matches',
|
|
869
976
|
transforms: [...appliedTransformNames],
|
|
977
|
+
policy: capabilityTransformPolicy,
|
|
870
978
|
};
|
|
871
979
|
|
|
872
980
|
const layerPipelineResult = buildLayerPipelineFromExecution(resolvedSteps, {
|
|
@@ -914,6 +1022,9 @@ export function compileExecutionV1(options = {}) {
|
|
|
914
1022
|
// until applyModelBatchingRuntimeDefaults in phase 2. If runtime batching was
|
|
915
1023
|
// already explicitly configured, manifest decodeLoop is skipped and runtime
|
|
916
1024
|
// values take precedence.
|
|
1025
|
+
// 3. preserveConfiguredKernelPath — restores a non-null runtime kernelPath and
|
|
1026
|
+
// its dtype-bearing config after compilation so explicit runtime config
|
|
1027
|
+
// remains the highest-precedence path contract.
|
|
917
1028
|
//
|
|
918
1029
|
// This function must be called exactly once per model load. Calling it again with
|
|
919
1030
|
// an already-patched runtimeConfig would double-apply the execution-v1 merge and
|
|
@@ -954,8 +1065,11 @@ export function applyExecutionV1RuntimeConfig(options = {}) {
|
|
|
954
1065
|
});
|
|
955
1066
|
|
|
956
1067
|
const runtimeInferencePatch = executionV1State.runtimeInferencePatch;
|
|
957
|
-
const updatedInference =
|
|
958
|
-
|
|
1068
|
+
const updatedInference = preserveConfiguredKernelPath(
|
|
1069
|
+
preserveRuntimeDecodeLoop(
|
|
1070
|
+
mergeRuntimeValues(runtimeConfig.inference ?? {}, runtimeInferencePatch),
|
|
1071
|
+
runtimeConfig
|
|
1072
|
+
),
|
|
959
1073
|
runtimeConfig
|
|
960
1074
|
);
|
|
961
1075
|
|
|
@@ -17,6 +17,8 @@ import { acquireBuffer, releaseBuffer, readBuffer } from '../../../../memory/buf
|
|
|
17
17
|
import {
|
|
18
18
|
runFusedFFN,
|
|
19
19
|
recordFusedFFN,
|
|
20
|
+
runFusedFFNFromRMSNormStats,
|
|
21
|
+
recordFusedFFNFromRMSNormStats,
|
|
20
22
|
castF16ToF32,
|
|
21
23
|
castF32ToF16,
|
|
22
24
|
recordCastF16ToF32,
|
|
@@ -31,6 +33,7 @@ import { getWeightBuffer, getNormWeightBuffer } from '../weights.js';
|
|
|
31
33
|
import { runProbes } from '../probes.js';
|
|
32
34
|
import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
33
35
|
import {
|
|
36
|
+
getKernelPathMatmulConstants,
|
|
34
37
|
getKernelPathMatmulPrecision,
|
|
35
38
|
getKernelPathMatmulVariant,
|
|
36
39
|
} from '../../../../config/kernel-path-loader.js';
|
|
@@ -54,6 +57,93 @@ function isQ4KMatmulVariant(variant) {
|
|
|
54
57
|
return typeof variant === 'string' && variant.startsWith('q4_');
|
|
55
58
|
}
|
|
56
59
|
|
|
60
|
+
function normalizeFusedGateUpPipelineConstants(constants) {
|
|
61
|
+
if (!constants || typeof constants !== 'object') {
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
const colsPerWorkgroup = constants.COLS_PER_WG;
|
|
65
|
+
const threadsPerCol = constants.THREADS_PER_COL ?? constants.THREADS_PER_COL_GEMV;
|
|
66
|
+
const workgroupSize = constants.WORKGROUP_SIZE;
|
|
67
|
+
const useFullBlockFastPath = constants.USE_FULL_BLOCK_FAST_PATH;
|
|
68
|
+
if (
|
|
69
|
+
colsPerWorkgroup === undefined &&
|
|
70
|
+
threadsPerCol === undefined &&
|
|
71
|
+
workgroupSize === undefined &&
|
|
72
|
+
useFullBlockFastPath === undefined
|
|
73
|
+
) {
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
return {
|
|
77
|
+
...(workgroupSize !== undefined ? { WORKGROUP_SIZE: workgroupSize } : {}),
|
|
78
|
+
...(colsPerWorkgroup !== undefined ? { COLS_PER_WG: colsPerWorkgroup } : {}),
|
|
79
|
+
...(threadsPerCol !== undefined ? { THREADS_PER_COL: threadsPerCol } : {}),
|
|
80
|
+
...(useFullBlockFastPath !== undefined ? { USE_FULL_BLOCK_FAST_PATH: useFullBlockFastPath } : {}),
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function constantsEqual(a, b) {
|
|
85
|
+
const aKeys = Object.keys(a).sort();
|
|
86
|
+
const bKeys = Object.keys(b).sort();
|
|
87
|
+
if (aKeys.length !== bKeys.length) return false;
|
|
88
|
+
for (let i = 0; i < aKeys.length; i++) {
|
|
89
|
+
if (aKeys[i] !== bKeys[i] || a[aKeys[i]] !== b[bKeys[i]]) {
|
|
90
|
+
return false;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return true;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function resolveFusedGateUpPipelineConstants(options = {}) {
|
|
97
|
+
const phase = options.phase ?? null;
|
|
98
|
+
const layerIdx = Number.isFinite(options.layerIdx) ? options.layerIdx : 0;
|
|
99
|
+
const kernelPath = options.kernelPath ?? null;
|
|
100
|
+
if (!phase) {
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
if (kernelPath) {
|
|
104
|
+
const fusedConstants = normalizeFusedGateUpPipelineConstants(
|
|
105
|
+
getKernelPathMatmulConstants('ffn_gate_up', phase, layerIdx, kernelPath)
|
|
106
|
+
);
|
|
107
|
+
if (fusedConstants) {
|
|
108
|
+
return fusedConstants;
|
|
109
|
+
}
|
|
110
|
+
const gateConstants = normalizeFusedGateUpPipelineConstants(
|
|
111
|
+
getKernelPathMatmulConstants('ffn_gate', phase, layerIdx, kernelPath)
|
|
112
|
+
);
|
|
113
|
+
const upConstants = normalizeFusedGateUpPipelineConstants(
|
|
114
|
+
getKernelPathMatmulConstants('ffn_up', phase, layerIdx, kernelPath)
|
|
115
|
+
);
|
|
116
|
+
if (gateConstants || upConstants) {
|
|
117
|
+
if (!gateConstants || !upConstants || !constantsEqual(gateConstants, upConstants)) {
|
|
118
|
+
throw new Error(
|
|
119
|
+
`[FFN] Fused gate/up requires matching gate and up kernel constants; ` +
|
|
120
|
+
`got gate=${JSON.stringify(gateConstants)} up=${JSON.stringify(upConstants)}.`
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
return gateConstants;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
const sessionConstants = getRuntimeConfig()
|
|
127
|
+
.inference?.session?.fusedFfnQ4K?.[phase]?.pipelineConstants;
|
|
128
|
+
return normalizeFusedGateUpPipelineConstants(sessionConstants);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export function resolveFusedGateUpVariant(options = {}) {
|
|
132
|
+
const phase = options.phase ?? null;
|
|
133
|
+
if (!phase) {
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
const variant = getRuntimeConfig()
|
|
137
|
+
.inference?.session?.fusedFfnQ4K?.[phase]?.variant;
|
|
138
|
+
if (variant == null) {
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
if (typeof variant !== 'string' || variant.length === 0) {
|
|
142
|
+
throw new Error(`[FFN] fusedFfnQ4K.${phase}.variant must be a non-empty string or null.`);
|
|
143
|
+
}
|
|
144
|
+
return variant;
|
|
145
|
+
}
|
|
146
|
+
|
|
57
147
|
function enqueueRecordedDenseHealth(context, layerIdx, label, tensor, elementCount) {
|
|
58
148
|
const recorder = context.recorder ?? null;
|
|
59
149
|
if (!recorder || !isTraceEnabled('logits') || !shouldDebugLayerOutput(layerIdx, context.debugLayers)) {
|
|
@@ -265,6 +355,31 @@ export function resolveGateUpPathMode(options = {}) {
|
|
|
265
355
|
return 'implicit';
|
|
266
356
|
}
|
|
267
357
|
|
|
358
|
+
export function canFuseSplitPrefillF16GateUpPath(options = {}) {
|
|
359
|
+
const kernelPath = options.kernelPath ?? null;
|
|
360
|
+
const phase = options.phase ?? null;
|
|
361
|
+
const layerIdx = Number.isFinite(options.layerIdx) ? options.layerIdx : 0;
|
|
362
|
+
if (!kernelPath || phase !== 'prefill') {
|
|
363
|
+
return false;
|
|
364
|
+
}
|
|
365
|
+
if (options.gateDtype !== 'f16' || options.upDtype !== 'f16') {
|
|
366
|
+
return false;
|
|
367
|
+
}
|
|
368
|
+
if (
|
|
369
|
+
hasExplicitMatmulPrecision('ffn_gate', phase, layerIdx, kernelPath)
|
|
370
|
+
|| hasExplicitMatmulPrecision('ffn_up', phase, layerIdx, kernelPath)
|
|
371
|
+
|| hasExplicitMatmulPrecision('ffn_down', phase, layerIdx, kernelPath)
|
|
372
|
+
) {
|
|
373
|
+
return false;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
const gateVariant = getKernelPathMatmulVariant('ffn_gate', phase, layerIdx, kernelPath);
|
|
377
|
+
const upVariant = getKernelPathMatmulVariant('ffn_up', phase, layerIdx, kernelPath);
|
|
378
|
+
return gateVariant != null
|
|
379
|
+
&& gateVariant === upVariant
|
|
380
|
+
&& !isQ4KMatmulVariant(gateVariant);
|
|
381
|
+
}
|
|
382
|
+
|
|
268
383
|
export function resolveFusedGateUpWeights(layerWeights, options = {}) {
|
|
269
384
|
const gate = layerWeights?.gate ?? null;
|
|
270
385
|
const up = layerWeights?.up ?? null;
|
|
@@ -319,6 +434,9 @@ async function dispatchFusedGateUp({
|
|
|
319
434
|
swigluLimit,
|
|
320
435
|
recorder,
|
|
321
436
|
executionPolicies = null,
|
|
437
|
+
normStats = null,
|
|
438
|
+
pipelineConstants = null,
|
|
439
|
+
variant = null,
|
|
322
440
|
}) {
|
|
323
441
|
const useNativeF16Fused = canUseNativeF16FusedGateUp({
|
|
324
442
|
inputDtype: inputTensor.dtype,
|
|
@@ -344,16 +462,62 @@ async function dispatchFusedGateUp({
|
|
|
344
462
|
}
|
|
345
463
|
|
|
346
464
|
const activation = resolveActivationOp(hiddenActivation);
|
|
465
|
+
if (normStats) {
|
|
466
|
+
if (variant != null) {
|
|
467
|
+
throw new Error(`[FFN] explicit fused gate/up variant "${variant}" is not supported by the norm-stats path.`);
|
|
468
|
+
}
|
|
469
|
+
if (inputTensor.dtype !== 'f32' || gateDtype !== 'q4k' || numTokens !== 1) {
|
|
470
|
+
throw new Error(
|
|
471
|
+
`Fused post-attention norm gate/up requires f32 Q4_K decode input; ` +
|
|
472
|
+
`got input=${inputTensor.dtype}, gate=${gateDtype}, tokens=${numTokens}.`
|
|
473
|
+
);
|
|
474
|
+
}
|
|
475
|
+
const fusedNormedOutput = recorder
|
|
476
|
+
? await recordFusedFFNFromRMSNormStats(
|
|
477
|
+
recorder,
|
|
478
|
+
inputTensor,
|
|
479
|
+
normStats.invRmsBuffer,
|
|
480
|
+
normStats.normWeight,
|
|
481
|
+
gateWeight,
|
|
482
|
+
upWeight,
|
|
483
|
+
hiddenSize,
|
|
484
|
+
intermediateSize,
|
|
485
|
+
{
|
|
486
|
+
batchSize: numTokens,
|
|
487
|
+
activation,
|
|
488
|
+
swigluLimit,
|
|
489
|
+
rmsNormWeightOffset: normStats.rmsNormWeightOffset === true,
|
|
490
|
+
pipelineConstants,
|
|
491
|
+
}
|
|
492
|
+
)
|
|
493
|
+
: await runFusedFFNFromRMSNormStats(
|
|
494
|
+
inputTensor,
|
|
495
|
+
normStats.invRmsBuffer,
|
|
496
|
+
normStats.normWeight,
|
|
497
|
+
gateWeight,
|
|
498
|
+
upWeight,
|
|
499
|
+
hiddenSize,
|
|
500
|
+
intermediateSize,
|
|
501
|
+
{
|
|
502
|
+
batchSize: numTokens,
|
|
503
|
+
activation,
|
|
504
|
+
swigluLimit,
|
|
505
|
+
rmsNormWeightOffset: normStats.rmsNormWeightOffset === true,
|
|
506
|
+
pipelineConstants,
|
|
507
|
+
}
|
|
508
|
+
);
|
|
509
|
+
return fusedNormedOutput;
|
|
510
|
+
}
|
|
347
511
|
const fusedOutput = recorder
|
|
348
512
|
? await recordFusedFFN(
|
|
349
513
|
recorder, fusedInput, gateWeight, upWeight,
|
|
350
514
|
hiddenSize, intermediateSize,
|
|
351
|
-
{ batchSize: numTokens, activation, swigluLimit }
|
|
515
|
+
{ batchSize: numTokens, activation, swigluLimit, pipelineConstants, variant }
|
|
352
516
|
)
|
|
353
517
|
: await runFusedFFN(
|
|
354
518
|
fusedInput, gateWeight, upWeight,
|
|
355
519
|
hiddenSize, intermediateSize,
|
|
356
|
-
{ batchSize: numTokens, activation, swigluLimit }
|
|
520
|
+
{ batchSize: numTokens, activation, swigluLimit, pipelineConstants, variant }
|
|
357
521
|
);
|
|
358
522
|
|
|
359
523
|
if (!recorder && fusedInput !== inputTensor) {
|
|
@@ -621,18 +785,29 @@ export async function runDenseFFNGPU(
|
|
|
621
785
|
hasQ4KMaterialization: fusedGateUpWeights.hasQ4KMaterialization,
|
|
622
786
|
activationDtype,
|
|
623
787
|
f16BatchSupported,
|
|
788
|
+
useLargeBatchF16F32FusedGateUp: context.useLargeBatchF16F32FusedGateUp === true,
|
|
624
789
|
batchSize: numTokens,
|
|
625
790
|
hiddenSizeAligned32,
|
|
626
791
|
useDoubleWideMlp: Boolean(useDoubleWideMlp),
|
|
627
792
|
});
|
|
628
|
-
const
|
|
793
|
+
const splitPrefillF16FusionAllowed = gateUpPathMode === 'split'
|
|
794
|
+
&& canFuseSplitPrefillF16GateUpPath({
|
|
795
|
+
kernelPath,
|
|
796
|
+
phase,
|
|
797
|
+
layerIdx,
|
|
798
|
+
gateDtype,
|
|
799
|
+
upDtype,
|
|
800
|
+
});
|
|
801
|
+
const useFusedGateUp = gateUpPathMode === 'split' && !splitPrefillF16FusionAllowed
|
|
629
802
|
? false
|
|
630
803
|
: useFusedGateUpByRule;
|
|
631
804
|
trace.ffn(
|
|
632
805
|
layerIdx,
|
|
633
|
-
`useFusedGateUp=${useFusedGateUp} gateUpPathMode=${gateUpPathMode}
|
|
806
|
+
`useFusedGateUp=${useFusedGateUp} gateUpPathMode=${gateUpPathMode} splitPrefillF16FusionAllowed=${splitPrefillF16FusionAllowed} ` +
|
|
807
|
+
`inputDtype=${inputTensor.dtype} activationDtype=${activationDtype} ` +
|
|
634
808
|
`gateDtype=${gateDtype} upDtype=${upDtype} hasQ4KMaterialization=${fusedGateUpWeights.hasQ4KMaterialization} ` +
|
|
635
|
-
`dtypeMatches=${dtypeMatches} dtypeSupported=${dtypeSupported} hiddenSizeAligned32=${hiddenSizeAligned32}
|
|
809
|
+
`dtypeMatches=${dtypeMatches} dtypeSupported=${dtypeSupported} hiddenSizeAligned32=${hiddenSizeAligned32} ` +
|
|
810
|
+
`largeBatchF16F32FusedGateUp=${context.useLargeBatchF16F32FusedGateUp === true} batchSize=${numTokens}`
|
|
636
811
|
);
|
|
637
812
|
|
|
638
813
|
if (useFusedGateUp) {
|
|
@@ -650,6 +825,12 @@ export async function runDenseFFNGPU(
|
|
|
650
825
|
const gateWeight = getWeightBuffer(fusedGateUpWeights.gate ?? layerWeights.gate, 'ffn_gate');
|
|
651
826
|
const upWeight = getWeightBuffer(fusedGateUpWeights.up ?? layerWeights.up, 'ffn_up');
|
|
652
827
|
const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
|
|
828
|
+
const fusedGateUpPipelineConstants = resolveFusedGateUpPipelineConstants({
|
|
829
|
+
kernelPath,
|
|
830
|
+
phase,
|
|
831
|
+
layerIdx,
|
|
832
|
+
});
|
|
833
|
+
const fusedGateUpVariant = resolveFusedGateUpVariant({ phase });
|
|
653
834
|
const fusedDownOutputDtype = resolveMatmulStepDtype(
|
|
654
835
|
'ffn_down',
|
|
655
836
|
phase,
|
|
@@ -674,6 +855,18 @@ export async function runDenseFFNGPU(
|
|
|
674
855
|
hiddenSize, intermediateSize, numTokens,
|
|
675
856
|
hiddenActivation, swigluLimit, recorder,
|
|
676
857
|
executionPolicies: context.executionPolicies ?? null,
|
|
858
|
+
normStats: context.__pendingFfnInputNormStats ?? null,
|
|
859
|
+
pipelineConstants: fusedGateUpPipelineConstants,
|
|
860
|
+
variant: fusedGateUpVariant,
|
|
861
|
+
});
|
|
862
|
+
await runProbes('ffn_act', fusedOutput.buffer, {
|
|
863
|
+
layerIdx,
|
|
864
|
+
numTokens,
|
|
865
|
+
hiddenSize: intermediateSize,
|
|
866
|
+
probes: context.debugProbes,
|
|
867
|
+
recorder,
|
|
868
|
+
operatorDiagnostics: context.operatorDiagnostics,
|
|
869
|
+
dtype: fusedOutput.dtype,
|
|
677
870
|
});
|
|
678
871
|
enqueueRecordedDenseHealth(context, layerIdx, 'ffn_fused_gate_up', fusedOutput, numTokens * intermediateSize);
|
|
679
872
|
|
|
@@ -778,6 +971,12 @@ export async function runDenseFFNGPU(
|
|
|
778
971
|
return output;
|
|
779
972
|
}
|
|
780
973
|
|
|
974
|
+
if (context.__pendingFfnInputNormStats) {
|
|
975
|
+
throw new Error(
|
|
976
|
+
`Layer ${layerIdx} has precomputed FFN input norm stats, but fused gate/up was not selected.`
|
|
977
|
+
);
|
|
978
|
+
}
|
|
979
|
+
|
|
781
980
|
if (!layerWeights?.gate || !layerWeights?.up || !layerWeights?.down) {
|
|
782
981
|
log.warn('Layer', `L${layerIdx} FFN: no weights found`);
|
|
783
982
|
const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: inputTensor.dtype });
|
|
@@ -1340,30 +1539,47 @@ export async function runDenseFFNWithFusedPostNormGPU(
|
|
|
1340
1539
|
hasQ4KMaterialization: fusedGateUpWeights.hasQ4KMaterialization,
|
|
1341
1540
|
activationDtype,
|
|
1342
1541
|
f16BatchSupported: getKernelCapabilities().hasF16,
|
|
1542
|
+
useLargeBatchF16F32FusedGateUp: context.useLargeBatchF16F32FusedGateUp === true,
|
|
1343
1543
|
batchSize: numTokens,
|
|
1344
1544
|
hiddenSizeAligned32,
|
|
1345
1545
|
useDoubleWideMlp: Boolean(useDoubleWideMlp),
|
|
1346
1546
|
});
|
|
1347
|
-
const
|
|
1547
|
+
const splitPrefillF16FusionAllowed = gateUpPathMode === 'split'
|
|
1548
|
+
&& canFuseSplitPrefillF16GateUpPath({
|
|
1549
|
+
kernelPath,
|
|
1550
|
+
phase,
|
|
1551
|
+
layerIdx,
|
|
1552
|
+
gateDtype,
|
|
1553
|
+
upDtype,
|
|
1554
|
+
});
|
|
1555
|
+
const canUseFusedGateUp = gateUpPathMode === 'split' && !splitPrefillF16FusionAllowed
|
|
1348
1556
|
? false
|
|
1349
1557
|
: canUseFusedGateUpByRule;
|
|
1350
1558
|
trace.ffn(
|
|
1351
1559
|
layerIdx,
|
|
1352
|
-
`useFusedGateUpWithPostNorm=${canUseFusedGateUp} gateUpPathMode=${gateUpPathMode}
|
|
1560
|
+
`useFusedGateUpWithPostNorm=${canUseFusedGateUp} gateUpPathMode=${gateUpPathMode} splitPrefillF16FusionAllowed=${splitPrefillF16FusionAllowed} ` +
|
|
1561
|
+
`inputDtype=${inputTensor.dtype} activationDtype=${activationDtype} ` +
|
|
1353
1562
|
`gateDtype=${gateDtype} upDtype=${upDtype} hasQ4KMaterialization=${fusedGateUpWeights.hasQ4KMaterialization} ` +
|
|
1354
|
-
`dtypeMatches=${dtypeMatches} dtypeSupported=${dtypeSupported} hiddenSizeAligned32=${hiddenSizeAligned32}
|
|
1563
|
+
`dtypeMatches=${dtypeMatches} dtypeSupported=${dtypeSupported} hiddenSizeAligned32=${hiddenSizeAligned32} ` +
|
|
1564
|
+
`largeBatchF16F32FusedGateUp=${context.useLargeBatchF16F32FusedGateUp === true} batchSize=${numTokens}`
|
|
1355
1565
|
);
|
|
1356
1566
|
const gateWeight = canUseFusedGateUp
|
|
1357
1567
|
? fusedGateWeight
|
|
1358
1568
|
: getWeightBuffer(layerWeights.gate, 'ffn_gate');
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1569
|
+
const upWeight = canUseFusedGateUp
|
|
1570
|
+
? fusedUpWeight
|
|
1571
|
+
: getWeightBuffer(layerWeights.up, 'ffn_up');
|
|
1572
|
+
|
|
1573
|
+
if (canUseFusedGateUp) {
|
|
1574
|
+
const fusedGateUpPipelineConstants = resolveFusedGateUpPipelineConstants({
|
|
1575
|
+
kernelPath,
|
|
1576
|
+
phase,
|
|
1577
|
+
layerIdx,
|
|
1578
|
+
});
|
|
1579
|
+
const fusedGateUpVariant = resolveFusedGateUpVariant({ phase });
|
|
1580
|
+
const {
|
|
1581
|
+
fusedGateUpInputDtype,
|
|
1582
|
+
} = resolveDenseFFNFusedPathDtypes({
|
|
1367
1583
|
phase,
|
|
1368
1584
|
layerIdx,
|
|
1369
1585
|
kernelPath,
|
|
@@ -1382,11 +1598,13 @@ export async function runDenseFFNWithFusedPostNormGPU(
|
|
|
1382
1598
|
fusedInputOwned = fusedInput !== inputTensor;
|
|
1383
1599
|
}
|
|
1384
1600
|
activatedOutput = await dispatchFusedGateUp({
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1601
|
+
inputTensor: fusedInput, gateWeight, upWeight, gateDtype,
|
|
1602
|
+
hiddenSize, intermediateSize, numTokens,
|
|
1603
|
+
hiddenActivation, swigluLimit, recorder,
|
|
1604
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
1605
|
+
pipelineConstants: fusedGateUpPipelineConstants,
|
|
1606
|
+
variant: fusedGateUpVariant,
|
|
1607
|
+
});
|
|
1390
1608
|
if (fusedInputOwned) {
|
|
1391
1609
|
if (recorder) {
|
|
1392
1610
|
recorder.trackTemporaryBuffer(fusedInput.buffer);
|