doppler-gpu 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +100 -38
- package/README.md +47 -195
- package/models/catalog.json +108 -51
- package/package.json +99 -123
- package/src/capsule-runtime.d.ts +17 -8
- package/src/capsule-runtime.js +8 -3
- package/src/cli/cli-output.js +85 -9
- package/src/cli/doppler-cli.js +23 -21
- package/src/client/capsule-host.browser.js +7 -0
- package/src/client/capsule-host.d.ts +13 -0
- package/src/client/capsule-host.js +7 -0
- package/src/client/doppler-api.browser.js +1 -1
- package/src/client/doppler-api.d.ts +3 -0
- package/src/client/doppler-api.js +2 -1
- package/src/client/doppler-registry.js +6 -0
- package/src/client/inspection.d.ts +10 -1
- package/src/client/inspection.js +24 -12
- package/src/client/model-host/index.d.ts +3 -0
- package/src/client/model-host/index.js +73 -18
- package/src/client/model-host/model-session.d.ts +42 -17
- package/src/client/model-host/model-session.js +291 -150
- package/src/client/model-host/source-loading.js +910 -0
- package/src/client/provider.js +6 -6
- package/src/client/resident-partitions.d.ts +30 -0
- package/src/client/resident-partitions.js +114 -0
- package/src/client/runtime/capsule-acquisition.d.ts +17 -2
- package/src/client/runtime/capsule-artifact-source.js +23 -1
- package/src/client/runtime/capsule-operation-adapters.d.ts +3 -2
- package/src/client/runtime/capsule-operation-adapters.js +55 -18
- package/src/client/runtime/capsule-operation-deltas.js +44 -0
- package/src/client/runtime/capsule-operation-executor.d.ts +2 -2
- package/src/client/runtime/capsule-operation-executor.js +14 -7
- package/src/client/runtime/capsule-operation-stream.d.ts +10 -0
- package/src/client/runtime/capsule-operation-stream.js +81 -0
- package/src/client/runtime/capsule-piece-reader.js +88 -0
- package/src/client/runtime/capsule-program-adapter.js +51 -19
- package/src/client/runtime/capsule-session-execution.js +9 -5
- package/src/client/runtime/command-executor.d.ts +13 -0
- package/src/client/runtime/command-executor.js +54 -11
- package/src/client/runtime/composition-root.d.ts +41 -15
- package/src/client/runtime/composition-root.js +198 -61
- package/src/client/runtime/fetch-capsule-artifact-store.d.ts +3 -2
- package/src/client/runtime/fetch-capsule-artifact-store.js +56 -1
- package/src/client/runtime/lora.d.ts +22 -0
- package/src/client/runtime/lora.js +86 -26
- package/src/client/runtime/model-manager.js +10 -75
- package/src/client/runtime/model-source.d.ts +1 -0
- package/src/client/runtime/model-source.js +1 -1
- package/src/client/runtime/node-quickstart-cache.js +7 -2
- package/src/client/runtime/resource-binder.d.ts +1 -0
- package/src/client/runtime/resource-binder.js +20 -14
- package/src/client/runtime/scoped-session.d.ts +10 -0
- package/src/client/runtime/scoped-session.js +41 -3
- package/src/client/runtime/session-controller.d.ts +19 -16
- package/src/client/runtime/session-controller.js +128 -134
- package/src/client/runtime/verified-capsule-artifact-store.d.ts +27 -0
- package/src/client/runtime/verified-capsule-artifact-store.js +266 -40
- package/src/client/wrap-pipeline-handle.js +23 -17
- package/src/config/canonical-contract-codec.d.ts +10 -0
- package/src/config/canonical-contract-codec.js +144 -0
- package/src/config/canonical-generation-policy.d.ts +7 -0
- package/src/config/canonical-generation-policy.js +66 -0
- package/src/config/capsule-loading.d.ts +5 -1
- package/src/config/capsule-loading.js +1 -1
- package/src/config/capsule-loading.schema.json +63 -3
- package/src/config/capsule-operation.d.ts +10 -3
- package/src/config/capsule-operation.js +12 -2
- package/src/config/capsule-operations.json +27 -3
- package/src/config/capsule-release-contract.js +2 -29
- package/src/config/capsule-token-selection.d.ts +15 -0
- package/src/config/capsule-token-selection.js +39 -0
- package/src/config/capsule-token-selection.json +11 -0
- package/src/config/capsule-v2.d.ts +1 -0
- package/src/config/capsule-v2.js +14 -21
- package/src/config/choice-scoring-reference.js +66 -0
- package/src/config/choice-scoring.d.ts +21 -0
- package/src/config/choice-scoring.js +66 -0
- package/src/config/choice-scoring.json +8 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +4 -4
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +9 -9
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +42 -20
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +260 -55
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +17 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +10 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +7 -7
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +10 -10
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +9 -8
- package/src/config/conversion/gemma3/gemma-3-270m-it-w8a8-ehf16-af32.json +491 -0
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +6 -6
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +10 -9
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +26 -10
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +26 -10
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +29 -13
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +22 -6
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +28 -11
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +28 -11
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +28 -11
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +26 -9
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +26 -9
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +8 -7
- package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +7 -7
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +8 -7
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +8 -7
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +20 -12
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +7 -9
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +7 -7
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +7 -7
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +252 -53
- package/src/config/conversion/qwen3/qwen-3-4b-instruct-2507-q4k-ehf16-af32.json +501 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-canonical-f32.json +461 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +11 -11
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +11 -11
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +7 -7
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +11 -11
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +11 -11
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +11 -11
- package/src/config/conversion/qwen3/qwen-3-6-35b-a3b-q4_0-passthrough.json +1218 -0
- package/src/config/conversion/qwen3/qwen-3-6-35b-a3b-text-q4k-ehf16-af32.json +581 -0
- package/src/config/conversion/qwen3/qwen-3-8-flash-next-text-q4k-ehf16-af32.json +1063 -0
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +11 -11
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +18 -10
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +10 -10
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +18 -18
- package/src/config/conversion/qwen3/qwen3.8-27b-q4_0-passthrough.json +823 -0
- package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +11 -11
- package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +11 -11
- package/src/config/embedding-contract.d.ts +17 -0
- package/src/config/embedding-contract.js +12 -0
- package/src/config/embedding-reference.js +32 -0
- package/src/config/execution-registry-contract.d.ts +41 -0
- package/src/config/execution-registry-contract.js +171 -0
- package/src/config/generation-contract.d.ts +66 -0
- package/src/config/generation-contract.js +66 -0
- package/src/config/generation-contract.json +151 -0
- package/src/config/initial-execution-identity.d.ts +1 -1
- package/src/config/initial-execution-identity.js +9 -9
- package/src/config/inspection/activity-stages.json +14 -0
- package/src/config/inspection/observation-policies.json +123 -0
- package/src/config/kernel-path-loader.d.ts +37 -0
- package/src/config/kernel-path-loader.js +87 -12
- package/src/config/kernel-registry-contract.d.ts +78 -13
- package/src/config/kernel-registry-contract.js +110 -38
- package/src/config/kernels/backward-registry.json +1 -1
- package/src/config/kernels/registry.js +3 -47
- package/src/config/kernels/registry.json +31642 -7719
- package/src/config/layer-partition-plan.d.ts +93 -0
- package/src/config/layer-partition-plan.js +305 -0
- package/src/config/manifest-inference-validation.js +10 -0
- package/src/config/merge/execution-patches.js +14 -0
- package/src/config/merge.d.ts +4 -0
- package/src/config/merge.js +50 -0
- package/src/config/model-ir-v2.d.ts +1 -0
- package/src/config/model-ir-v2.js +5 -16
- package/src/config/model-ir.js +9 -14
- package/src/config/ordered-resident-qualification.d.ts +16 -0
- package/src/config/ordered-resident-qualification.js +62 -0
- package/src/config/param-categories.js +4 -0
- package/src/config/param-validator.js +64 -29
- package/src/config/production-release-evidence.js +2 -27
- package/src/config/production-release.js +2 -29
- package/src/config/quickstart-registry.json +57 -8
- package/src/config/reasoning-output.d.ts +6 -0
- package/src/config/reasoning-output.js +26 -0
- package/src/config/revocation-policy.js +1 -4
- package/src/config/rope-frequencies.js +9 -0
- package/src/config/runtime/profiles/demo-f16-comparison.json +20 -0
- package/src/config/runtime/profiles/flash-next-node-host-cache-qualification.json +21 -0
- package/src/config/runtime/profiles/flash-next-paged-qualification.json +23 -0
- package/src/config/runtime/profiles/flash-next-routed-f32kv-qualification.json +72 -0
- package/src/config/runtime/profiles/flash-next-routed-qualification.json +27 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +51 -2
- package/src/config/runtime/profiles/qwen-3-5-0-8b-vulkan-parity.json +88 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8.json +173 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-vulkan-throughput.json +33 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +51 -2
- package/src/config/runtime/profiles/qwen-3-5-2b-vulkan-parity.json +15 -0
- package/src/config/runtime/profiles/qwen3-6-35b-a3b-q4_0-candidate.json +4491 -0
- package/src/config/runtime/profiles/qwen3-6-35b-a3b-q4_0-pooled-q8-common-diagnostic.json +4407 -0
- package/src/config/runtime/profiles/qwen3-6-35b-a3b-route-f32-candidate.json +17 -0
- package/src/config/runtime/profiles/qwen3-6-35b-a3b-route-f32-workspace-candidate.json +13 -0
- package/src/config/runtime.d.ts +6 -0
- package/src/config/runtime.js +51 -21
- package/src/config/schema/benchmark.schema.d.ts +1 -0
- package/src/config/schema/benchmark.schema.js +2 -0
- package/src/config/schema/conversion.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.d.ts +10 -0
- package/src/config/schema/converter.schema.js +10 -0
- package/src/config/schema/debug.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +17 -9
- package/src/config/schema/emulation.schema.js +0 -77
- package/src/config/schema/execution-v1.schema.d.ts +32 -2
- package/src/config/schema/execution-v1.schema.js +8 -0
- package/src/config/schema/fp8.schema.d.ts +17 -0
- package/src/config/schema/fp8.schema.js +27 -0
- package/src/config/schema/hyper-connection-decoder.schema.d.ts +98 -0
- package/src/config/schema/hyper-connection-decoder.schema.js +213 -0
- package/src/config/schema/index.d.ts +2 -2
- package/src/config/schema/index.js +3 -0
- package/src/config/schema/inference.schema.d.ts +8 -2
- package/src/config/schema/kernel-registry.schema.d.ts +22 -7
- package/src/config/schema/kernel-registry.schema.js +13 -2
- package/src/config/schema/kernel-thresholds.schema.d.ts +3 -0
- package/src/config/schema/kernel-thresholds.schema.js +14 -3
- package/src/config/schema/kernel-warmup.schema.d.ts +15 -0
- package/src/config/schema/kernel-warmup.schema.js +3 -0
- package/src/config/schema/loading.schema.d.ts +41 -0
- package/src/config/schema/loading.schema.js +67 -0
- package/src/config/schema/manifest-rope.schema.d.ts +55 -0
- package/src/config/schema/manifest.schema.d.ts +47 -57
- package/src/config/schema/manifest.schema.js +41 -2
- package/src/config/schema/ngram-embedding.schema.d.ts +116 -0
- package/src/config/schema/ngram-embedding.schema.js +312 -0
- package/src/config/schema/program-bundle.schema.d.ts +1 -0
- package/src/config/schema/program-bundle.schema.js +8 -3
- package/src/config/schema/quantization-defaults.schema.d.ts +1 -1
- package/src/config/schema/quantization.schema.d.ts +2 -0
- package/src/config/schema/quantization.schema.js +2 -0
- package/src/config/schema/speculation-self.schema.d.ts +6 -1
- package/src/config/schema/speculation-self.schema.js +29 -2
- package/src/config/schema/w8a8.schema.d.ts +24 -0
- package/src/config/schema/w8a8.schema.js +38 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/supported-operations.json +105 -0
- package/src/config/target-plan.d.ts +16 -2
- package/src/config/target-plan.js +63 -16
- package/src/config/wgsl-language-contract.js +47 -0
- package/src/converter/artifact-identity.js +27 -47
- package/src/converter/capability-transform-resolver.js +80 -0
- package/src/converter/conversion-plan.js +14 -14
- package/src/converter/core.d.ts +2 -0
- package/src/converter/core.js +15 -16
- package/src/converter/execution-graph/fusion.js +343 -0
- package/src/converter/execution-graph/kv-storage.js +14 -0
- package/src/converter/execution-graph/validation.js +431 -0
- package/src/converter/execution-graph-transforms.js +927 -0
- package/src/converter/forge-candidate-evaluation.js +1 -1
- package/src/converter/forge-qualification.js +103 -31
- package/src/converter/forge-stages.js +113 -106
- package/src/converter/fp8-source.js +99 -0
- package/src/converter/linear-norm-gate.js +115 -0
- package/src/converter/manifest-builder.js +20 -6
- package/src/converter/parsers/gguf.js +21 -2
- package/src/converter/quantization-info.js +56 -5
- package/src/converter/quantizer.js +4 -145
- package/src/converter/qwen4-exp-source-defaults.json +35 -0
- package/src/converter/qwen4-exp-source.js +293 -0
- package/src/converter/resident-qualification.js +58 -0
- package/src/converter/retained-source-shards.js +97 -0
- package/src/converter/rope-config.js +1 -0
- package/src/converter/source-rotary-frequencies.d.ts +12 -0
- package/src/converter/source-rotary-frequencies.js +34 -0
- package/src/converter/source-tensor-rules.d.ts +25 -1
- package/src/converter/source-tensor-rules.js +31 -1
- package/src/converter/tensor-transform.js +60 -7
- package/src/converter/tokenizer-contract.js +1 -3
- package/src/converter/w8a8-quantizer.js +56 -0
- package/src/debug/activation-trace.d.ts +34 -0
- package/src/debug/activation-trace.js +89 -0
- package/src/debug/activity-trace.d.ts +52 -0
- package/src/debug/activity-trace.js +127 -0
- package/src/debug/capture-policy.d.ts +9 -1
- package/src/debug/capture-policy.js +52 -21
- package/src/debug/index.d.ts +3 -0
- package/src/debug/index.js +12 -6
- package/src/debug/log.d.ts +6 -0
- package/src/debug/log.js +26 -0
- package/src/debug/perf-guards.js +71 -0
- package/src/debug/pipeline-config.js +141 -0
- package/src/debug/stats.d.ts +1 -1
- package/src/debug/stats.js +7 -2
- package/src/debug/tensor.d.ts +15 -3
- package/src/debug/tensor.js +43 -27
- package/src/debug/trace.js +12 -0
- package/src/errors/doppler-error.js +6 -2
- package/src/experimental/audio/gemma4.js +19 -10
- package/src/experimental/bdpa/basis-decomposed-paged.d.ts +1 -1
- package/src/experimental/diffusion/pipeline.d.ts +1 -0
- package/src/experimental/distribution/shard-delivery/receipt.js +3 -73
- package/src/experimental/distribution/shard-delivery/retry.js +6 -112
- package/src/experimental/distribution/shard-delivery.js +19 -632
- package/src/experimental/energy/pipeline.d.ts +1 -0
- package/src/experimental/logits/cpu.js +2 -7
- package/src/experimental/training/artifacts.js +1 -4
- package/src/experimental/training/attention-forward.js +1 -1
- package/src/experimental/training/autograd.d.ts +21 -0
- package/src/experimental/training/autograd.js +44 -20
- package/src/experimental/training/checkpoint.js +1 -4
- package/src/experimental/training/clip.js +1 -1
- package/src/experimental/training/distillation/student-fixture/execution.js +9 -12
- package/src/experimental/training/distillation/student-fixture/model.js +9 -12
- package/src/experimental/training/distillation/student-fixture.js +9 -12
- package/src/experimental/training/lora/checkpoint.js +1 -1
- package/src/experimental/training/lora/execution.js +1 -1
- package/src/experimental/training/lora/plan.js +1 -1
- package/src/experimental/training/lora/recovery.js +1 -1
- package/src/experimental/training/lora-pipeline.js +9 -83
- package/src/experimental/training/lora-training-round.d.ts +39 -0
- package/src/experimental/training/lora-training-round.js +90 -0
- package/src/experimental/training/lora.js +2 -1
- package/src/experimental/training/loss-scaling.js +1 -1
- package/src/experimental/training/loss.js +3 -1
- package/src/experimental/training/native-qwen-lora.d.ts +25 -6
- package/src/experimental/training/native-qwen-lora.js +176 -21
- package/src/experimental/training/operator-command.js +1 -49
- package/src/experimental/training/parameter-groups.js +39 -0
- package/src/experimental/training/report-ranking.js +52 -0
- package/src/experimental/training/runner/lifecycle.js +4 -61
- package/src/experimental/training/runner.js +2 -5
- package/src/experimental/training/suite/execution.js +3 -5
- package/src/experimental/training/suite/plan.js +6 -21
- package/src/experimental/training/suite.js +5 -34
- package/src/experimental/training/trainer.js +2 -36
- package/src/experimental/training/workload-capsules/registry.json +473 -0
- package/src/experimental/turboquant/codebook.js +0 -29
- package/src/formats/contract-validation.js +30 -0
- package/src/formats/flatbuffer-tables.js +55 -0
- package/src/formats/float16.js +81 -0
- package/src/formats/float8.js +19 -0
- package/src/formats/gguf/block-storage.js +51 -0
- package/src/formats/gguf/quantized-formats.json +10 -0
- package/src/formats/gguf/qwen-source.js +88 -0
- package/src/formats/gguf/qwen-source.json +8 -0
- package/src/formats/gguf/tokenizer-pre.json +4 -0
- package/src/formats/gguf/tokenizer.js +50 -0
- package/src/formats/litert/types.js +8 -180
- package/src/formats/paged-source-reader.js +158 -0
- package/src/formats/q4k-dequant.js +95 -0
- package/src/formats/rdrr/classification.js +4 -0
- package/src/formats/rdrr/distributed/validation.js +1 -4
- package/src/formats/rdrr/fp8-contract.js +38 -0
- package/src/formats/rdrr/fp8-storage.js +83 -0
- package/src/formats/rdrr/functional-descriptor.js +1 -4
- package/src/formats/rdrr/parsing.js +1 -1
- package/src/formats/rdrr/q4_0-storage.js +24 -0
- package/src/formats/rdrr/storage-descriptor.js +7 -1
- package/src/formats/rdrr/types.d.ts +7 -1
- package/src/formats/rdrr/validation.js +37 -6
- package/src/formats/rdrr/w8a8-storage.js +64 -0
- package/src/formats/safetensors/types.js +4 -0
- package/src/formats/sha256.js +104 -103
- package/src/formats/sorted-json.js +9 -0
- package/src/formats/source-runtime.d.ts +52 -0
- package/src/formats/tflite/types.js +7 -161
- package/src/generation/index.d.ts +7 -1
- package/src/generation/index.js +7 -1
- package/src/gpu/buffer-identity.js +4 -0
- package/src/gpu/command-recorder.d.ts +66 -29
- package/src/gpu/command-recorder.js +157 -161
- package/src/gpu/device-bindings.js +98 -0
- package/src/gpu/device-state.js +0 -4
- package/src/gpu/device.d.ts +9 -0
- package/src/gpu/device.js +105 -137
- package/src/gpu/kernel-selector.js +73 -12
- package/src/gpu/kernels/attention/executors-components.js +235 -0
- package/src/gpu/kernels/attention/executors.js +296 -510
- package/src/gpu/kernels/attention/plan.js +65 -18
- package/src/gpu/kernels/attention/query-splitk.js +80 -0
- package/src/gpu/kernels/attention_decode_gqa_splitk.wgsl +197 -0
- package/src/gpu/kernels/attention_decode_online.wgsl +348 -0
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +126 -22
- package/src/gpu/kernels/attention_decode_subgroup.wgsl +5 -6
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +56 -9
- package/src/gpu/kernels/attention_indexed.wgsl +179 -0
- package/src/gpu/kernels/attention_indexed_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_paired.wgsl +361 -0
- package/src/gpu/kernels/attention_paired_tiled.wgsl +383 -0
- package/src/gpu/kernels/attention_prefill_partitioned_head256_f16kv.wgsl +247 -0
- package/src/gpu/kernels/attention_prefill_query_splitk_f32.wgsl +234 -0
- package/src/gpu/kernels/attention_prefill_query_splitk_reduce_f32.wgsl +103 -0
- package/src/gpu/kernels/attention_prefill_query_tile_f16kv.wgsl +225 -0
- package/src/gpu/kernels/attention_prefill_query_tile_f32.wgsl +222 -0
- package/src/gpu/kernels/backward/adam.js +11 -3
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +51 -15
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +9 -8
- package/src/gpu/kernels/backward/gated_delta_recurrent_checkpoint_forward.js +149 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_checkpoint_forward.wgsl +109 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_checkpointed_backward.js +120 -0
- package/src/gpu/kernels/backward/gelu_backward.js +8 -12
- package/src/gpu/kernels/backward/index.js +6 -0
- package/src/gpu/kernels/backward/qwen_attention_split_q_gate_backward.js +80 -0
- package/src/gpu/kernels/backward/qwen_attention_split_q_gate_backward.wgsl +26 -0
- package/src/gpu/kernels/backward/qwen_gqa_attention_backward.js +157 -0
- package/src/gpu/kernels/backward/qwen_gqa_attention_backward.wgsl +82 -0
- package/src/gpu/kernels/backward/qwen_gqa_softmax_backward_scores.wgsl +73 -0
- package/src/gpu/kernels/backward/qwen_gqa_softmax_recompute.wgsl +97 -0
- package/src/gpu/kernels/backward/qwen_linear_attention_prepare_backward.js +221 -0
- package/src/gpu/kernels/backward/qwen_linear_attention_prepare_backward.wgsl +136 -0
- package/src/gpu/kernels/backward/rmsnorm_backward.js +98 -80
- package/src/gpu/kernels/backward/rope_backward.js +47 -10
- package/src/gpu/kernels/backward/rope_backward.wgsl +46 -22
- package/src/gpu/kernels/backward/sigmoid_gated_backward.js +83 -0
- package/src/gpu/kernels/backward/sigmoid_gated_backward.wgsl +49 -0
- package/src/gpu/kernels/backward/silu_backward.js +8 -12
- package/src/gpu/kernels/backward/softmax_backward.js +8 -12
- package/src/gpu/kernels/cast_f32_to_two_f32.wgsl +12 -0
- package/src/gpu/kernels/cast_two_f32_to_f16.wgsl +49 -0
- package/src/gpu/kernels/causal_conv1d_silu.js +80 -0
- package/src/gpu/kernels/causal_conv1d_silu.wgsl +44 -0
- package/src/gpu/kernels/check-stop.js +1 -1
- package/src/gpu/kernels/check_finiteness_f32.wgsl +33 -0
- package/src/gpu/kernels/conv2d.js +30 -42
- package/src/gpu/kernels/dequant.js +1 -1
- package/src/gpu/kernels/dequant_mxfp4.wgsl +6 -4
- package/src/gpu/kernels/diffusion-gemma-sampling.js +26 -7
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +66 -2
- package/src/gpu/kernels/dispatch.js +36 -1
- package/src/gpu/kernels/embedding-pool.js +15 -13
- package/src/gpu/kernels/feature-check.js +7 -2
- package/src/gpu/kernels/fp8.js +70 -0
- package/src/gpu/kernels/fused_ffn.js +83 -154
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +43 -1
- package/src/gpu/kernels/fused_ffn_q4k_subgroup8.wgsl +228 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +2 -2
- package/src/gpu/kernels/fused_matmul_q4_subgroup.wgsl +146 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +14 -4
- package/src/gpu/kernels/fused_matmul_q4_widetile_direct.wgsl +204 -0
- package/src/gpu/kernels/fused_matmul_residual.js +19 -72
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +34 -127
- package/src/gpu/kernels/gated-delta-norm-gate.js +45 -0
- package/src/gpu/kernels/gated_delta_conv.wgsl +57 -6
- package/src/gpu/kernels/gated_delta_conv_f16.wgsl +57 -6
- package/src/gpu/kernels/gated_delta_conv_paired.wgsl +282 -0
- package/src/gpu/kernels/gated_delta_conv_prefill.wgsl +144 -0
- package/src/gpu/kernels/gated_delta_fused_decode.wgsl +375 -40
- package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +375 -40
- package/src/gpu/kernels/gated_delta_fused_decode_paired.wgsl +511 -0
- package/src/gpu/kernels/gated_delta_fused_decode_paired_f16.wgsl +513 -0
- package/src/gpu/kernels/gated_delta_fused_decode_paired_sigmoid_gate.wgsl +514 -0
- package/src/gpu/kernels/gated_delta_fused_decode_paired_sigmoid_gate_f16.wgsl +516 -0
- package/src/gpu/kernels/gated_delta_fused_decode_register.wgsl +750 -0
- package/src/gpu/kernels/gated_delta_fused_decode_sigmoid_gate.wgsl +511 -0
- package/src/gpu/kernels/gated_delta_fused_decode_sigmoid_gate_f16.wgsl +513 -0
- package/src/gpu/kernels/gated_delta_gate_paired.wgsl +271 -0
- package/src/gpu/kernels/gated_delta_gate_sigmoid_paired.wgsl +273 -0
- package/src/gpu/kernels/gated_delta_norm_paired.wgsl +288 -0
- package/src/gpu/kernels/gated_delta_prefill_norm_gate.wgsl +227 -0
- package/src/gpu/kernels/gated_delta_qk_paired.wgsl +297 -0
- package/src/gpu/kernels/gated_delta_qk_paired_parallel.wgsl +299 -0
- package/src/gpu/kernels/gated_delta_recurrent.wgsl +362 -32
- package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +362 -32
- package/src/gpu/kernels/gated_delta_recurrent_gguf.wgsl +496 -0
- package/src/gpu/kernels/gated_delta_recurrent_gguf_register_prefill.wgsl +727 -0
- package/src/gpu/kernels/gated_delta_recurrent_paired.wgsl +337 -0
- package/src/gpu/kernels/gated_delta_recurrent_register_prefill.wgsl +546 -0
- package/src/gpu/kernels/gated_delta_recurrent_sigmoid_gate.wgsl +498 -0
- package/src/gpu/kernels/gated_delta_recurrent_sigmoid_gate_f16.wgsl +500 -0
- package/src/gpu/kernels/gated_delta_state_paired.wgsl +310 -0
- package/src/gpu/kernels/gated_delta_state_paired_sliced.wgsl +348 -0
- package/src/gpu/kernels/gated_delta_state_paired_sliced_rolled.wgsl +353 -0
- package/src/gpu/kernels/gated_rmsnorm.js +74 -0
- package/src/gpu/kernels/gated_rmsnorm.wgsl +62 -0
- package/src/gpu/kernels/gather.js +81 -46
- package/src/gpu/kernels/gather.wgsl +3 -2
- package/src/gpu/kernels/gather_bf16.wgsl +3 -2
- package/src/gpu/kernels/gather_f16.wgsl +3 -2
- package/src/gpu/kernels/gather_f16_f16_out.wgsl +3 -2
- package/src/gpu/kernels/gather_f16_out.wgsl +3 -2
- package/src/gpu/kernels/gather_f16_vec4.wgsl +3 -2
- package/src/gpu/kernels/gather_f16_vec4_f16_out.wgsl +3 -2
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +3 -2
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +3 -2
- package/src/gpu/kernels/gather_q4_0.wgsl +57 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +3 -2
- package/src/gpu/kernels/gather_split4_f16_vec4_f32_out.wgsl +104 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +3 -2
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +3 -2
- package/src/gpu/kernels/gather_vec4.wgsl +3 -2
- package/src/gpu/kernels/gather_vec4_f16_out.wgsl +3 -2
- package/src/gpu/kernels/gelu.js +16 -13
- package/src/gpu/kernels/gelu.wgsl +16 -0
- package/src/gpu/kernels/gelu_f16.wgsl +16 -0
- package/src/gpu/kernels/gemma4-route-expert.js +2 -5
- package/src/gpu/kernels/generated/uniform-writers-0.js +600 -0
- package/src/gpu/kernels/generated/uniform-writers-1.js +602 -0
- package/src/gpu/kernels/generated/uniform-writers-2.js +516 -0
- package/src/gpu/kernels/generated/uniform-writers.js +557 -0
- package/src/gpu/kernels/hyper-connection.js +240 -0
- package/src/gpu/kernels/hyper_connection_activation.wgsl +110 -0
- package/src/gpu/kernels/hyper_connection_branch.wgsl +60 -0
- package/src/gpu/kernels/hyper_connection_inject.wgsl +49 -0
- package/src/gpu/kernels/hyper_connection_norm.wgsl +72 -0
- package/src/gpu/kernels/kernel-bindings.js +37 -0
- package/src/gpu/kernels/kernel-configs.d.ts +12 -2
- package/src/gpu/kernels/kernel-configs.js +40 -23
- package/src/gpu/kernels/kernel-execution.js +87 -26
- package/src/gpu/kernels/kernel-geometry.js +43 -0
- package/src/gpu/kernels/kernel-prewarm.js +18 -15
- package/src/gpu/kernels/kv-cache-write.js +4 -6
- package/src/gpu/kernels/layernorm.js +7 -45
- package/src/gpu/kernels/linear-attention-core.js +247 -119
- package/src/gpu/kernels/linear-attention-recovery.js +90 -0
- package/src/gpu/kernels/linear-norm-ab.js +80 -0
- package/src/gpu/kernels/linear_norm_ab_f32.wgsl +243 -0
- package/src/gpu/kernels/logit-finalize.js +9 -3
- package/src/gpu/kernels/logit-scatter.js +9 -3
- package/src/gpu/kernels/logit-suppress.js +24 -0
- package/src/gpu/kernels/logit_suppress.wgsl +19 -0
- package/src/gpu/kernels/matmul-dispatch.js +70 -146
- package/src/gpu/kernels/matmul-selection.js +75 -18
- package/src/gpu/kernels/matmul-workgroup-capacity.js +37 -0
- package/src/gpu/kernels/matmul.js +81 -108
- package/src/gpu/kernels/matmul_compensated_f32.wgsl +215 -0
- package/src/gpu/kernels/matmul_f16w_f32a_tiled.wgsl +59 -46
- package/src/gpu/kernels/matmul_f32.wgsl +23 -17
- package/src/gpu/kernels/matmul_f32_row_subgroup.wgsl +88 -0
- package/src/gpu/kernels/matmul_fp8.wgsl +60 -0
- package/src/gpu/kernels/matmul_gemv_compensated_f32.wgsl +211 -0
- package/src/gpu/kernels/matmul_gemv_f32.wgsl +63 -0
- package/src/gpu/kernels/matmul_gemv_f32_subgroup.wgsl +84 -0
- package/src/gpu/kernels/matmul_gguf_columns_hoisted_f32.wgsl +342 -0
- package/src/gpu/kernels/matmul_gguf_pair_columns_f32.wgsl +279 -0
- package/src/gpu/kernels/matmul_gguf_pair_f32.wgsl +197 -0
- package/src/gpu/kernels/matmul_paired.wgsl +315 -0
- package/src/gpu/kernels/matmul_paired_compensated.wgsl +334 -0
- package/src/gpu/kernels/matmul_q4_0.wgsl +234 -0
- package/src/gpu/kernels/matmul_q4_0_tile128.wgsl +477 -0
- package/src/gpu/kernels/matmul_q4_0_tile128_bounded.wgsl +483 -0
- package/src/gpu/kernels/matmul_q4_0_tile64.wgsl +282 -0
- package/src/gpu/kernels/matmul_q4_0_tile64_bounded.wgsl +288 -0
- package/src/gpu/kernels/matmul_q4_0_tile64x128.wgsl +364 -0
- package/src/gpu/kernels/matmul_q4_0_tile64x128_bounded.wgsl +370 -0
- package/src/gpu/kernels/matmul_q4_1_columns_f32.wgsl +132 -0
- package/src/gpu/kernels/matmul_q5k.wgsl +181 -0
- package/src/gpu/kernels/matmul_q5k_columns.wgsl +194 -0
- package/src/gpu/kernels/matmul_q5k_cooperative.wgsl +129 -0
- package/src/gpu/kernels/matmul_q5k_tile64.wgsl +309 -0
- package/src/gpu/kernels/matmul_w8a8.wgsl +55 -0
- package/src/gpu/kernels/matmul_w8a8_packed.wgsl +54 -0
- package/src/gpu/kernels/matmul_w8a8_packed_cooperative.wgsl +77 -0
- package/src/gpu/kernels/moe-grouped.js +132 -0
- package/src/gpu/kernels/moe-norm-router.js +91 -0
- package/src/gpu/kernels/moe-route.js +135 -0
- package/src/gpu/kernels/moe-shared-slot.js +219 -0
- package/src/gpu/kernels/moe_grouped_matmul_f32.wgsl +216 -0
- package/src/gpu/kernels/moe_grouped_matmul_ordered_f32.wgsl +158 -0
- package/src/gpu/kernels/moe_grouped_sort.wgsl +101 -0
- package/src/gpu/kernels/moe_norm_router_f32.wgsl +254 -0
- package/src/gpu/kernels/moe_route_gguf_split_f32.wgsl +117 -0
- package/src/gpu/kernels/moe_route_q4_0_f32.wgsl +81 -0
- package/src/gpu/kernels/moe_route_q4_f32.wgsl +250 -0
- package/src/gpu/kernels/moe_shared_slot_combine_f32.wgsl +91 -0
- package/src/gpu/kernels/moe_shared_slot_down_f32.wgsl +264 -0
- package/src/gpu/kernels/moe_shared_slot_down_pair_combine_f32.wgsl +254 -0
- package/src/gpu/kernels/moe_shared_slot_down_pair_combine_residual_f32.wgsl +262 -0
- package/src/gpu/kernels/moe_shared_slot_down_pair_f32.wgsl +187 -0
- package/src/gpu/kernels/moe_shared_slot_gu_f32.wgsl +264 -0
- package/src/gpu/kernels/moe_shared_slot_gu_pair_f32.wgsl +267 -0
- package/src/gpu/kernels/paired-activation.js +86 -0
- package/src/gpu/kernels/paired-attention.js +85 -0
- package/src/gpu/kernels/paired-cast.js +82 -0
- package/src/gpu/kernels/paired-linear-attention.js +180 -0
- package/src/gpu/kernels/paired-matmul.js +86 -0
- package/src/gpu/kernels/paired-rmsnorm.js +127 -0
- package/src/gpu/kernels/paired-rope.js +44 -0
- package/src/gpu/kernels/paired-split.js +74 -0
- package/src/gpu/kernels/paired_add.wgsl +261 -0
- package/src/gpu/kernels/paired_multiply.wgsl +261 -0
- package/src/gpu/kernels/paired_scale.wgsl +260 -0
- package/src/gpu/kernels/paired_scale_inplace.wgsl +259 -0
- package/src/gpu/kernels/paired_sigmoid.wgsl +266 -0
- package/src/gpu/kernels/paired_silu.wgsl +266 -0
- package/src/gpu/kernels/paired_silu_unary.wgsl +260 -0
- package/src/gpu/kernels/pipeline-cache.d.ts +37 -4
- package/src/gpu/kernels/pipeline-cache.js +185 -88
- package/src/gpu/kernels/ple.js +145 -0
- package/src/gpu/kernels/ple_conv.wgsl +104 -0
- package/src/gpu/kernels/ple_gate.wgsl +131 -0
- package/src/gpu/kernels/q4_0.js +200 -0
- package/src/gpu/kernels/qsa-indexer.d.ts +105 -0
- package/src/gpu/kernels/qsa-indexer.js +304 -0
- package/src/gpu/kernels/qsa_block_keys.wgsl +110 -0
- package/src/gpu/kernels/qsa_block_scores.wgsl +73 -0
- package/src/gpu/kernels/qsa_prepare.wgsl +116 -0
- package/src/gpu/kernels/qsa_select.wgsl +172 -0
- package/src/gpu/kernels/quantize_w8a8.wgsl +81 -0
- package/src/gpu/kernels/qwen_attention_split_q_gate.js +71 -0
- package/src/gpu/kernels/qwen_attention_split_q_gate.wgsl +26 -0
- package/src/gpu/kernels/qwen_linear_attention_decay_paired.wgsl +266 -0
- package/src/gpu/kernels/qwen_linear_attention_log_decay_paired.wgsl +273 -0
- package/src/gpu/kernels/qwen_linear_attention_prepare.js +153 -0
- package/src/gpu/kernels/qwen_linear_attention_prepare.wgsl +120 -0
- package/src/gpu/kernels/qwen_linear_attention_prepare_paired.wgsl +267 -0
- package/src/gpu/kernels/qwen_linear_attention_scale_paired.wgsl +276 -0
- package/src/gpu/kernels/relu.js +5 -12
- package/src/gpu/kernels/rep-penalty.js +30 -5
- package/src/gpu/kernels/rep_penalty.wgsl +25 -30
- package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
- package/src/gpu/kernels/repeat_channels.js +22 -9
- package/src/gpu/kernels/repeat_channels.wgsl +12 -5
- package/src/gpu/kernels/repeat_channels_f16.wgsl +12 -5
- package/src/gpu/kernels/rescale_w8a8.wgsl +74 -0
- package/src/gpu/kernels/residual.js +5 -0
- package/src/gpu/kernels/rmsnorm-pair.js +2 -2
- package/src/gpu/kernels/rmsnorm-qk.js +27 -5
- package/src/gpu/kernels/rmsnorm-stats.js +62 -78
- package/src/gpu/kernels/rmsnorm.js +74 -96
- package/src/gpu/kernels/rmsnorm.wgsl +9 -3
- package/src/gpu/kernels/rmsnorm_f16.wgsl +2 -0
- package/src/gpu/kernels/rmsnorm_paired.wgsl +321 -0
- package/src/gpu/kernels/rmsnorm_paired_sliced.wgsl +326 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +32 -8
- package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +7 -7
- package/src/gpu/kernels/rope-dispatch.js +26 -0
- package/src/gpu/kernels/rope-precompute.js +29 -28
- package/src/gpu/kernels/rope-qk.js +35 -24
- package/src/gpu/kernels/rope.js +5 -2
- package/src/gpu/kernels/rope.wgsl +2 -2
- package/src/gpu/kernels/rope_precompute.wgsl +150 -6
- package/src/gpu/kernels/rope_precompute_paired.wgsl +285 -0
- package/src/gpu/kernels/rope_qk.wgsl +3 -2
- package/src/gpu/kernels/rope_qk_paired.wgsl +341 -0
- package/src/gpu/kernels/sample.js +135 -249
- package/src/gpu/kernels/sample.wgsl +138 -168
- package/src/gpu/kernels/sample_f16.wgsl +192 -171
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scatter_add_routes_f32.wgsl +43 -0
- package/src/gpu/kernels/shader-cache.d.ts +7 -1
- package/src/gpu/kernels/shader-cache.js +61 -20
- package/src/gpu/kernels/shader-source-scope.d.ts +14 -0
- package/src/gpu/kernels/shader-source-scope.js +127 -12
- package/src/gpu/kernels/sigmoid-gated-add.js +68 -0
- package/src/gpu/kernels/sigmoid_gated_add_f32.wgsl +76 -0
- package/src/gpu/kernels/silu.js +56 -136
- package/src/gpu/kernels/silu.wgsl +54 -3
- package/src/gpu/kernels/silu_f16.wgsl +3 -2
- package/src/gpu/kernels/softmax.js +57 -19
- package/src/gpu/kernels/softmax_topk_logit_rank.wgsl +137 -0
- package/src/gpu/kernels/softmax_topk_ranked.wgsl +132 -0
- package/src/gpu/kernels/softmax_topk_strided.wgsl +80 -0
- package/src/gpu/kernels/speculative-accept.js +97 -0
- package/src/gpu/kernels/speculative_accept.wgsl +38 -0
- package/src/gpu/kernels/speculative_accept_gpu_limit.wgsl +42 -0
- package/src/gpu/kernels/split-dispatch.js +18 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +3 -6
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +4 -11
- package/src/gpu/kernels/split_qg.js +6 -2
- package/src/gpu/kernels/split_qg.wgsl +3 -2
- package/src/gpu/kernels/split_qg_f16.wgsl +3 -2
- package/src/gpu/kernels/split_qg_paired.wgsl +59 -0
- package/src/gpu/kernels/split_qkv.js +7 -16
- package/src/gpu/kernels/split_qkv_paired.wgsl +75 -0
- package/src/gpu/kernels/subgroup-partition.js +11 -0
- package/src/gpu/kernels/uniform-encoding.js +20 -0
- package/src/gpu/kernels/uniform-utils.js +44 -49
- package/src/gpu/kernels/vision-average-pool.js +2 -5
- package/src/gpu/kernels/vision-patch-embed.js +2 -5
- package/src/gpu/kernels/vision-position-embedding.js +2 -5
- package/src/gpu/kernels/vision-rope-2d.js +2 -5
- package/src/gpu/kernels/vision-spatial-merge.js +2 -5
- package/src/gpu/kernels/w8a8.js +166 -0
- package/src/gpu/prepared-decode-workspace.d.ts +11 -0
- package/src/gpu/prepared-decode-workspace.js +104 -0
- package/src/gpu/recorder-timestamps.js +63 -0
- package/src/gpu/step-contract.d.ts +27 -0
- package/src/gpu/step-contract.js +26 -0
- package/src/gpu/submit-tracker.js +1 -17
- package/src/gpu/tensor.d.ts +13 -4
- package/src/gpu/tensor.js +38 -19
- package/src/gpu/uniform-cache.d.ts +8 -4
- package/src/gpu/uniform-cache.js +41 -25
- package/src/gpu/weight-buffer.d.ts +10 -1
- package/src/gpu/weight-buffer.js +2 -1
- package/src/index.d.ts +3 -0
- package/src/index.js +2 -0
- package/src/inference/choice-scoring.js +70 -0
- package/src/inference/decode-buffers.d.ts +11 -3
- package/src/inference/decode-buffers.js +93 -63
- package/src/inference/generation-output.js +128 -0
- package/src/inference/generation-step.d.ts +14 -0
- package/src/inference/generation-step.js +26 -0
- package/src/inference/kv-cache/base.d.ts +7 -2
- package/src/inference/kv-cache/base.js +10 -8
- package/src/inference/kv-cache/copy-state.js +39 -0
- package/src/inference/kv-cache/layer-range.js +76 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +3 -2
- package/src/inference/kv-cache/mixed-geometry.js +25 -1
- package/src/inference/kv-cache/quantized.d.ts +1 -1
- package/src/inference/kv-cache/quantized.js +1 -1
- package/src/inference/kv-cache/sliding-window.d.ts +2 -2
- package/src/inference/kv-cache/sliding-window.js +58 -183
- package/src/inference/kv-cache/tiered.d.ts +1 -1
- package/src/inference/kv-cache/tiered.js +1 -1
- package/src/inference/kv-cache/types.js +2 -28
- package/src/inference/kv-cache/update.js +104 -157
- package/src/inference/moe-router.d.ts +5 -0
- package/src/inference/moe-router.js +42 -31
- package/src/inference/pipelines/audio/index.js +5 -2
- package/src/inference/pipelines/audio/mel.js +116 -81
- package/src/inference/pipelines/context.js +110 -76
- package/src/inference/pipelines/factory.js +27 -5
- package/src/inference/pipelines/registry.js +9 -3
- package/src/inference/pipelines/shader-scoped-pipeline.d.ts +24 -0
- package/src/inference/pipelines/shader-scoped-pipeline.js +243 -11
- package/src/inference/pipelines/shared/clipped-linear.js +1 -1
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +115 -118
- package/src/inference/pipelines/structured/json-head-pipeline.js +1 -1
- package/src/inference/pipelines/text/activity-weights.js +110 -0
- package/src/inference/pipelines/text/attention/cache-cast.js +15 -0
- package/src/inference/pipelines/text/attention/dispatch-params.js +2 -1
- package/src/inference/pipelines/text/attention/interpreter/immediate.js +16 -19
- package/src/inference/pipelines/text/attention/interpreter/recorded.js +20 -21
- package/src/inference/pipelines/text/attention/interpreter.js +34 -38
- package/src/inference/pipelines/text/attention/paired-contract.js +29 -0
- package/src/inference/pipelines/text/attention/plan.d.ts +2 -0
- package/src/inference/pipelines/text/attention/plan.js +11 -4
- package/src/inference/pipelines/text/attention/projection-executor.js +10 -27
- package/src/inference/pipelines/text/attention/projection-plan.js +24 -31
- package/src/inference/pipelines/text/attention/projections.js +10 -27
- package/src/inference/pipelines/text/attention/query-transform.js +2 -2
- package/src/inference/pipelines/text/attention/recorded-health.js +2 -1
- package/src/inference/pipelines/text/attention/rope-observation.js +26 -3
- package/src/inference/pipelines/text/chat-format.js +2 -1
- package/src/inference/pipelines/text/config/normalization.js +115 -226
- package/src/inference/pipelines/text/config/vision-normalizers.js +240 -0
- package/src/inference/pipelines/text/config-components.js +49 -0
- package/src/inference/pipelines/text/config.d.ts +45 -1
- package/src/inference/pipelines/text/config.js +26 -38
- package/src/inference/pipelines/text/debug-utils/config.js +13 -156
- package/src/inference/pipelines/text/debug-utils/health-metrics.js +8 -23
- package/src/inference/pipelines/text/debug-utils/readback-decoding.js +5 -1
- package/src/inference/pipelines/text/embed.js +38 -4
- package/src/inference/pipelines/text/embedding-batch.js +23 -0
- package/src/inference/pipelines/text/embedding-extraction.js +7 -2
- package/src/inference/pipelines/text/embedding-normalization.js +2 -1
- package/src/inference/pipelines/text/execution-plan.d.ts +16 -0
- package/src/inference/pipelines/text/execution-plan.js +49 -2
- package/src/inference/pipelines/text/execution-runtime-builders.js +31 -0
- package/src/inference/pipelines/text/execution-v1/validation.js +3 -3
- package/src/inference/pipelines/text/execution-v1.js +14 -14
- package/src/inference/pipelines/text/execution.js +78 -128
- package/src/inference/pipelines/text/ffn/activation-observation.js +45 -0
- package/src/inference/pipelines/text/ffn/decode-intermediate.js +10 -0
- package/src/inference/pipelines/text/ffn/dense-executor.js +60 -50
- package/src/inference/pipelines/text/ffn/dense-plan.js +28 -27
- package/src/inference/pipelines/text/ffn/dense.js +30 -50
- package/src/inference/pipelines/text/ffn/moe.js +67 -3
- package/src/inference/pipelines/text/ffn/paired.js +144 -0
- package/src/inference/pipelines/text/ffn/qwen-shared-expert.js +212 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +8 -3
- package/src/inference/pipelines/text/ffn/standard.js +67 -16
- package/src/inference/pipelines/text/finiteness-guard-status.js +9 -0
- package/src/inference/pipelines/text/generation-request.d.ts +36 -0
- package/src/inference/pipelines/text/generation-request.js +73 -0
- package/src/inference/pipelines/text/generator/activation-storage.js +15 -0
- package/src/inference/pipelines/text/generator/batch-retry-policy.js +20 -0
- package/src/inference/pipelines/text/generator/capsule-generation.js +37 -0
- package/src/inference/pipelines/text/generator/compiled-greedy-window.js +154 -0
- package/src/inference/pipelines/text/generator/decode-components.js +24 -0
- package/src/inference/pipelines/text/generator/decode-readback.js +73 -0
- package/src/inference/pipelines/text/generator/decode-runtime.js +401 -403
- package/src/inference/pipelines/text/generator/decode.js +594 -626
- package/src/inference/pipelines/text/generator/diagnostic-sequence.d.ts +63 -0
- package/src/inference/pipelines/text/generator/diagnostic-sequence.js +106 -0
- package/src/inference/pipelines/text/generator/diffusion.js +24 -12
- package/src/inference/pipelines/text/generator/greedy-head-policy.js +15 -0
- package/src/inference/pipelines/text/generator/logits-config.js +11 -4
- package/src/inference/pipelines/text/generator/per-layer-buffer.js +21 -0
- package/src/inference/pipelines/text/generator/prefill-observation.js +25 -1
- package/src/inference/pipelines/text/generator/prefill-policy.js +30 -18
- package/src/inference/pipelines/text/generator/prefill-runtime.js +511 -511
- package/src/inference/pipelines/text/generator/recorded-step-executor.js +70 -0
- package/src/inference/pipelines/text/generator/recorder-stats.js +17 -0
- package/src/inference/pipelines/text/generator/recovery.js +6 -0
- package/src/inference/pipelines/text/generator/session-context.js +19 -7
- package/src/inference/pipelines/text/generator/speculative-runtime.js +82 -0
- package/src/inference/pipelines/text/generator/text.js +17 -149
- package/src/inference/pipelines/text/generator/token-selection.d.ts +7 -0
- package/src/inference/pipelines/text/generator/token-selection.js +59 -0
- package/src/inference/pipelines/text/generator/training.js +42 -60
- package/src/inference/pipelines/text/generator/whole-token-lane.js +371 -0
- package/src/inference/pipelines/text/generator/whole-token-observation.d.ts +72 -0
- package/src/inference/pipelines/text/generator/whole-token-observation.js +502 -0
- package/src/inference/pipelines/text/generator/whole-token-program.d.ts +93 -0
- package/src/inference/pipelines/text/generator/whole-token-program.js +731 -0
- package/src/inference/pipelines/text/generator/whole-token-template.js +437 -0
- package/src/inference/pipelines/text/generator-logits-observation.js +5 -2
- package/src/inference/pipelines/text/generator-runtime.d.ts +107 -0
- package/src/inference/pipelines/text/generator-runtime.js +38 -79
- package/src/inference/pipelines/text/generator-steps.js +62 -55
- package/src/inference/pipelines/text/generator.d.ts +10 -1
- package/src/inference/pipelines/text/generator.js +111 -175
- package/src/inference/pipelines/text/hyper-connection/decoder-layer.d.ts +75 -0
- package/src/inference/pipelines/text/hyper-connection/decoder-layer.js +132 -0
- package/src/inference/pipelines/text/hyper-connection/layer-executor.js +104 -0
- package/src/inference/pipelines/text/hyper-connection/logits-input.js +20 -0
- package/src/inference/pipelines/text/hyper-connection/projection.js +21 -0
- package/src/inference/pipelines/text/hyper-connection/sequence-transaction.js +145 -0
- package/src/inference/pipelines/text/hyper-connection/session.d.ts +39 -0
- package/src/inference/pipelines/text/hyper-connection/session.js +236 -0
- package/src/inference/pipelines/text/image-transcription.js +2 -3
- package/src/inference/pipelines/text/indexed-attention/attention-runner.js +49 -0
- package/src/inference/pipelines/text/indexed-attention/qsa-cache.d.ts +54 -0
- package/src/inference/pipelines/text/indexed-attention/qsa-cache.js +256 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +121 -104
- package/src/inference/pipelines/text/init-rope.d.ts +26 -0
- package/src/inference/pipelines/text/init-rope.js +109 -14
- package/src/inference/pipelines/text/init-tokenizer.js +29 -0
- package/src/inference/pipelines/text/init.d.ts +28 -10
- package/src/inference/pipelines/text/init.js +65 -74
- package/src/inference/pipelines/text/kernel-trace.js +33 -11
- package/src/inference/pipelines/text/layer-execution-contract.js +1 -1
- package/src/inference/pipelines/text/layer-input-execution.js +10 -9
- package/src/inference/pipelines/text/layer-partition-contract.d.ts +47 -0
- package/src/inference/pipelines/text/layer-partition-contract.js +220 -0
- package/src/inference/pipelines/text/layer-plan-gpu.js +153 -80
- package/src/inference/pipelines/text/layer-plan.js +9 -2
- package/src/inference/pipelines/text/layer.js +34 -47
- package/src/inference/pipelines/text/lifecycle.js +79 -178
- package/src/inference/pipelines/text/linear-attention/activity.js +29 -0
- package/src/inference/pipelines/text/linear-attention/norm-ab.js +59 -0
- package/src/inference/pipelines/text/linear-attention/plan.js +35 -34
- package/src/inference/pipelines/text/linear-attention/state-checkpoint.js +83 -0
- package/src/inference/pipelines/text/linear-attention/state-clone.js +109 -0
- package/src/inference/pipelines/text/linear-attention/state-layout.js +45 -0
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +48 -45
- package/src/inference/pipelines/text/linear-attention.d.ts +20 -0
- package/src/inference/pipelines/text/linear-attention.js +150 -131
- package/src/inference/pipelines/text/load-errors.js +2 -19
- package/src/inference/pipelines/text/load-policy.js +55 -0
- package/src/inference/pipelines/text/logits/gpu-executor.js +130 -103
- package/src/inference/pipelines/text/logits/gpu.js +93 -70
- package/src/inference/pipelines/text/logits/index.d.ts +14 -2
- package/src/inference/pipelines/text/logits/index.js +420 -352
- package/src/inference/pipelines/text/logits/output-transform.js +37 -25
- package/src/inference/pipelines/text/logits/precision-policy.js +1 -1
- package/src/inference/pipelines/text/logits/types.d.ts +8 -1
- package/src/inference/pipelines/text/lora-apply.js +1 -1
- package/src/inference/pipelines/text/model-load.js +71 -117
- package/src/inference/pipelines/text/moe/gpu-executor.js +39 -17
- package/src/inference/pipelines/text/moe/grouped-prefill.js +67 -0
- package/src/inference/pipelines/text/moe/qwen-route-executor.d.ts +41 -0
- package/src/inference/pipelines/text/moe/qwen-route-executor.js +125 -0
- package/src/inference/pipelines/text/moe/routed-expert-chunks.js +95 -0
- package/src/inference/pipelines/text/moe-gpu.js +132 -50
- package/src/inference/pipelines/text/moe-impl.d.ts +23 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +12 -0
- package/src/inference/pipelines/text/ngram/ngram-addresses.d.ts +30 -0
- package/src/inference/pipelines/text/ngram/ngram-addresses.js +134 -0
- package/src/inference/pipelines/text/ngram/ngram-table.d.ts +89 -0
- package/src/inference/pipelines/text/ngram/ngram-table.js +233 -0
- package/src/inference/pipelines/text/ngram/ple-layer.d.ts +54 -0
- package/src/inference/pipelines/text/ngram/ple-layer.js +115 -0
- package/src/inference/pipelines/text/operator-events.d.ts +129 -0
- package/src/inference/pipelines/text/operator-events.js +19 -2
- package/src/inference/pipelines/text/ops.js +40 -30
- package/src/inference/pipelines/text/paired-execution.js +140 -0
- package/src/inference/pipelines/text/partition-attempt.js +105 -0
- package/src/inference/pipelines/text/partition-execution.js +347 -0
- package/src/inference/pipelines/text/per-layer/materialize.js +3 -10
- package/src/inference/pipelines/text/per-layer/plan.js +3 -10
- package/src/inference/pipelines/text/per-layer-inputs.js +3 -10
- package/src/inference/pipelines/text/pipeline-load-timing.js +2 -1
- package/src/inference/pipelines/text/pipeline.d.ts +227 -0
- package/src/inference/pipelines/text/pipeline.js +629 -0
- package/src/inference/pipelines/text/precomputed-input-norm.js +29 -0
- package/src/inference/pipelines/text/probes.js +65 -23
- package/src/inference/pipelines/text/resident-partition-contract.d.ts +179 -0
- package/src/inference/pipelines/text/resident-partition-contract.js +98 -0
- package/src/inference/pipelines/text/resident-partition.js +403 -0
- package/src/inference/pipelines/text/resident-pipeline-preparation.js +341 -0
- package/src/inference/pipelines/text/sampling-config.d.ts +16 -0
- package/src/inference/pipelines/text/sampling-config.js +25 -113
- package/src/inference/pipelines/text/sampling.d.ts +9 -1
- package/src/inference/pipelines/text/sampling.js +18 -375
- package/src/inference/pipelines/text/sequence-state.d.ts +68 -0
- package/src/inference/pipelines/text/sequence-state.js +514 -0
- package/src/inference/pipelines/text/stage-names.d.ts +88 -0
- package/src/inference/pipelines/text/state.d.ts +17 -8
- package/src/inference/pipelines/text/state.js +10 -0
- package/src/inference/pipelines/text/stopping.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +77 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +3 -12
- package/src/inference/pipelines/text/types.d.ts +68 -114
- package/src/inference/pipelines/text/warm-generation.d.ts +27 -0
- package/src/inference/pipelines/text/warm-generation.js +34 -0
- package/src/inference/pipelines/text/weight-loading-config.js +16 -10
- package/src/inference/pipelines/text/weights.d.ts +103 -1
- package/src/inference/pipelines/text/weights.js +0 -17
- package/src/inference/pipelines/text.d.ts +3 -211
- package/src/inference/pipelines/text.js +16 -578
- package/src/inference/pipelines/video/feature-concat.js +30 -0
- package/src/inference/pipelines/video/frame-sampler.js +4 -0
- package/src/inference/pipelines/video/index.js +33 -44
- package/src/inference/pipelines/vision/encoder.js +1 -1
- package/src/inference/pipelines/vision/gemma4.js +6 -7
- package/src/inference/pipelines/vision/glmocr.js +11 -13
- package/src/inference/pipelines/vision/index.js +14 -3
- package/src/inference/pipelines/vision/support.js +93 -0
- package/src/inference/pipelines/vision/weight-loading.js +111 -0
- package/src/inference/rerank.d.ts +44 -0
- package/src/inference/resident-block-contract.d.ts +35 -0
- package/src/inference/resident-block-contract.js +105 -0
- package/src/inference/resident-output-state.d.ts +14 -0
- package/src/inference/resident-output-state.js +89 -0
- package/src/inference/resident-publication-contract.d.ts +32 -0
- package/src/inference/resident-publication-contract.js +104 -0
- package/src/inference/resident-state-contract.d.ts +26 -0
- package/src/inference/resident-state-contract.js +135 -0
- package/src/inference/resident-target-state.d.ts +32 -0
- package/src/inference/resident-target-state.js +152 -0
- package/src/inference/sampling-rng.js +35 -0
- package/src/inference/token-sampling.js +395 -0
- package/src/inference/tokenizer.d.ts +2 -0
- package/src/inference/tokenizer.js +16 -0
- package/src/inference/tokenizers/bundled/incremental-decoder.d.ts +16 -0
- package/src/inference/tokenizers/bundled/incremental-decoder.js +100 -0
- package/src/inference/tokenizers/bundled/normalizer.js +44 -0
- package/src/inference/tokenizers/bundled.js +25 -42
- package/src/inference/tokenizers/sentencepiece.js +4 -26
- package/src/loader/doppler-loader.d.ts +13 -0
- package/src/loader/doppler-loader.js +34 -62
- package/src/loader/dtype-utils.js +1 -19
- package/src/loader/embedding-limit-preflight.js +77 -8
- package/src/loader/embedding-loader.js +43 -114
- package/src/loader/experts/expert-cache.js +1 -4
- package/src/loader/experts/expert-loader.d.ts +6 -1
- package/src/loader/experts/expert-loader.js +90 -29
- package/src/loader/experts/expert-preparation.d.ts +16 -0
- package/src/loader/experts/expert-preparation.js +57 -0
- package/src/loader/experts/loader-bindings.js +53 -0
- package/src/loader/experts/qwen-paging.d.ts +19 -0
- package/src/loader/experts/qwen-paging.js +70 -0
- package/src/loader/experts/routed-expert-cache.d.ts +56 -0
- package/src/loader/experts/routed-expert-cache.js +372 -0
- package/src/loader/experts/routed-expert-plan.d.ts +20 -0
- package/src/loader/experts/routed-expert-plan.js +46 -0
- package/src/loader/final-weights-loader.d.ts +7 -1
- package/src/loader/final-weights-loader.js +17 -40
- package/src/loader/hyper-connection-loader.d.ts +25 -0
- package/src/loader/hyper-connection-loader.js +150 -0
- package/src/loader/layer-loader.d.ts +8 -1
- package/src/loader/layer-loader.js +67 -97
- package/src/loader/layer-tensor-aliases.js +77 -11
- package/src/loader/layer-weight-shape.js +13 -0
- package/src/loader/load-timing.js +2 -0
- package/src/loader/loader-types.d.ts +24 -1
- package/src/loader/model-executor.js +27 -365
- package/src/loader/model-load.d.ts +18 -0
- package/src/loader/model-load.js +432 -0
- package/src/loader/ngram-table-bindings.d.ts +11 -0
- package/src/loader/ngram-table-bindings.js +61 -0
- package/src/loader/per-layer-input-loader.js +5 -83
- package/src/loader/q4k-reference-weight.js +84 -0
- package/src/loader/shard-cache.js +3 -0
- package/src/loader/tensor-load-error.js +20 -1
- package/src/loader/tensors/bf16-loader.js +3 -29
- package/src/loader/tensors/buffer-write.js +26 -0
- package/src/loader/tensors/fp8-loader.js +28 -0
- package/src/loader/tensors/load-executor.js +124 -117
- package/src/loader/tensors/q4_0-loader.js +32 -0
- package/src/loader/tensors/range-backed-weights.js +34 -0
- package/src/loader/tensors/source-transform.js +1 -1
- package/src/loader/tensors/tensor-loader.js +22 -6
- package/src/loader/tensors/w8a8-loader.js +36 -0
- package/src/loader/weight-downcast.js +9 -13
- package/src/loader/weight-representation-plan.d.ts +28 -0
- package/src/loader/weight-representation-plan.js +146 -0
- package/src/loader/weights.d.ts +1 -1
- package/src/memory/buffer-pool.d.ts +24 -4
- package/src/memory/buffer-pool.js +124 -128
- package/src/memory/device-budget.d.ts +10 -0
- package/src/memory/device-budget.js +89 -0
- package/src/memory/device-residency.js +60 -0
- package/src/partitions.d.ts +21 -0
- package/src/partitions.js +10 -0
- package/src/rules/execution-rules-contract-check.js +73 -570
- package/src/rules/generated/rule-bundle.json +1 -4305
- package/src/rules/greedy-sampling.js +16 -0
- package/src/rules/inference/capability-transforms.rules.json +11 -1
- package/src/rules/layer-pattern-contract-check.js +49 -222
- package/src/rules/rule-matcher.d.ts +62 -0
- package/src/rules/rule-matcher.js +132 -1
- package/src/rules/rule-registry.d.ts +75 -0
- package/src/rules/rule-registry.js +64 -24
- package/src/storage/artifact-storage-context.d.ts +99 -0
- package/src/storage/artifact-storage-context.js +71 -114
- package/src/storage/backend-selection.js +3 -0
- package/src/storage/backends/idb-store.d.ts +54 -0
- package/src/storage/backends/idb-store.js +12 -0
- package/src/storage/backends/memory-store.d.ts +38 -0
- package/src/storage/backends/memory-store.js +15 -6
- package/src/storage/backends/opfs-store.d.ts +43 -0
- package/src/storage/backends/opfs-store.js +6 -0
- package/src/storage/blake3.d.ts +17 -0
- package/src/storage/blake3.js +153 -68
- package/src/storage/distribution-transport.js +626 -3
- package/src/storage/download/http-contract.js +199 -0
- package/src/storage/download/http-transport.js +824 -0
- package/src/storage/download/integrity.d.ts +10 -2
- package/src/storage/download/integrity.js +39 -2
- package/src/storage/download/transport.js +2 -2
- package/src/storage/downloader.js +404 -388
- package/src/storage/export.d.ts +1 -0
- package/src/storage/export.js +22 -70
- package/src/storage/model-cache.d.ts +4 -1
- package/src/storage/model-cache.js +131 -52
- package/src/storage/model-read-session.d.ts +13 -0
- package/src/storage/model-read-session.js +76 -0
- package/src/storage/node-file-access.js +108 -0
- package/src/storage/reports.js +1 -4
- package/src/storage/runtime-env.js +6 -0
- package/src/storage/shard-manager.d.ts +10 -0
- package/src/storage/shard-manager.js +644 -579
- package/src/storage/shards/index.js +16 -0
- package/src/storage/shards/integrity.js +35 -49
- package/src/storage/source-artifact-store.d.ts +59 -0
- package/src/storage/source-artifact-store.js +24 -37
- package/src/storage/source-storage-context.d.ts +53 -0
- package/src/storage/source-storage-context.js +298 -0
- package/src/storage/verified-piece-reader.d.ts +23 -0
- package/src/storage/verified-piece-reader.js +134 -0
- package/src/storage/verified-piece-storage.d.ts +7 -0
- package/src/storage/verified-piece-storage.js +16 -0
- package/src/tooling/boundary-evidence.js +1 -4
- package/src/tooling/browser-command-runner.d.ts +1 -1
- package/src/tooling/browser-command-runner.js +2 -2
- package/src/tooling/browser-harness/benchmark-comparison.js +105 -0
- package/src/tooling/browser-harness/benchmark-execution.js +865 -0
- package/src/tooling/browser-harness/decode-diagnostics.js +201 -0
- package/src/tooling/browser-harness/inference-execution.js +543 -0
- package/src/tooling/browser-harness/model-initialization.js +230 -0
- package/src/tooling/browser-harness/model-resolution.js +134 -0
- package/src/tooling/browser-harness/report.js +338 -0
- package/src/tooling/browser-harness/request-timing.js +50 -0
- package/src/tooling/browser-harness/request.js +264 -0
- package/src/tooling/browser-harness/runtime-config.d.ts +26 -0
- package/src/tooling/browser-harness/runtime-config.js +256 -0
- package/src/tooling/browser-harness/text-evidence.d.ts +81 -0
- package/src/tooling/browser-harness/text-evidence.js +695 -0
- package/src/tooling/browser-harness/text-execution.d.ts +37 -0
- package/src/tooling/browser-harness/text-execution.js +721 -0
- package/src/tooling/browser-harness/text-input.d.ts +45 -0
- package/src/tooling/browser-harness/text-input.js +722 -0
- package/src/tooling/browser-harness/timing-diagnostics.js +416 -0
- package/src/tooling/browser-harness-text-helpers.d.ts +31 -0
- package/src/tooling/browser-harness-text-helpers.js +31 -0
- package/src/tooling/browser-harness.d.ts +245 -0
- package/src/tooling/browser-harness.js +337 -0
- package/src/tooling/command-api.d.ts +1 -0
- package/src/tooling/diagnose-runner.js +2 -2
- package/src/tooling/distribution-shard-transport.d.ts +6 -0
- package/src/tooling/execution-cost-ledger.d.ts +2 -1
- package/src/tooling/execution-cost-ledger.js +10 -7
- package/src/tooling/hf-registry/catalog-io.js +7 -5
- package/src/tooling/litert-package-runtime.js +1 -98
- package/src/tooling/model-capsule-pieces.js +47 -0
- package/src/tooling/model-capsule-rig.js +528 -0
- package/src/tooling/node-browser/execution.d.ts +1 -1
- package/src/tooling/node-browser/static-server.js +26 -2
- package/src/tooling/node-browser/transport.js +0 -43
- package/src/tooling/node-capsule-artifact-store.js +24 -0
- package/src/tooling/node-command-execution.js +2 -2
- package/src/tooling/node-command-runner.d.ts +1 -1
- package/src/tooling/node-converter/input.js +36 -5
- package/src/tooling/node-converter/output.js +75 -1
- package/src/tooling/node-converter.js +33 -34
- package/src/tooling/node-source-runtime.js +5 -5
- package/src/tooling/portable-source-path.js +26 -0
- package/src/tooling/production-release.js +3 -6
- package/src/tooling/program-bundle/choice-scoring-reference.js +22 -0
- package/src/tooling/program-bundle/materialize.js +4 -0
- package/src/tooling/program-bundle/wgsl-closure.js +3 -9
- package/src/tooling/program-bundle-host.js +26 -0
- package/src/tooling/program-bundle.js +11 -6
- package/src/tooling/reference-verify.js +5 -17
- package/src/tooling/registered-variant-calibration-job.js +1 -4
- package/src/tooling/registered-variant-calibration.js +1 -4
- package/src/tooling/source-artifact-adapter.d.ts +1 -0
- package/src/tooling/source-artifact-adapter.js +33 -1
- package/src/tooling/source-intake.js +1 -4
- package/src/tooling/source-runtime-bundle.d.ts +4 -85
- package/src/tooling/source-runtime-bundle.js +3 -3
- package/src/tooling/source-runtime-converter-config.js +41 -8
- package/src/tooling/test-harness.d.ts +173 -0
- package/src/tooling/test-harness.js +476 -0
- package/src/tooling-exports/device.d.ts +1 -0
- package/src/tooling-exports/device.js +1 -0
- package/src/tooling-exports/evidence.d.ts +3 -0
- package/src/tooling-exports/manifest.d.ts +1 -1
- package/src/tooling-exports/manifest.js +1 -1
- package/src/tooling-exports/runtime.d.ts +6 -1
- package/src/tooling-exports/runtime.js +6 -1
- package/src/tooling-exports/storage.d.ts +10 -0
- package/src/tooling-exports/storage.js +9 -0
- package/src/tooling-exports/structured.d.ts +2 -1
- package/src/tooling-exports/structured.js +2 -1
- package/src/tooling-exports.shared.d.ts +12 -9
- package/src/tooling-exports.shared.js +6 -12
- package/src/training-browser.d.ts +5 -0
- package/src/training.d.ts +7 -0
- package/src/training.js +1 -0
- package/src/version.js +1 -1
- package/tests/fixtures/linear-core-activation-accuracy.json +12 -0
- package/tests/fixtures/linear-core-refined-accuracy.json +128 -0
- package/tests/fixtures/q4k-ordered-accumulation.json +87 -0
- package/tests/fixtures/rmsnorm-inverse-root.json +18 -0
- package/tests/kernels/browser/kernel-suite.js +230 -0
- package/tests/kernels/browser/test-page.js +5 -6
- package/tests/kernels/harness/benchmark.js +11 -3
- package/tests/kernels/reference/dequant.js +23 -3
- package/tests/kernels/reference/index.js +1 -0
- package/tools/convert-safetensors-node.js +7 -1
- package/src/capsule.d.ts +0 -8
- package/src/capsule.js +0 -4
- package/src/cli/serve/capsule-handler.d.ts +0 -33
- package/src/cli/serve/capsule-handler.js +0 -105
- package/src/cli/serve/capsule-http.js +0 -102
- package/src/client/runtime/index.js +0 -1
- package/src/client/runtime/source-runtime.js +0 -910
- package/src/client/runtime/target-selector.js +0 -2
- package/src/config/backward-registry-loader.d.ts +0 -3
- package/src/config/capsule-serve.d.ts +0 -9
- package/src/config/capsule-serve.js +0 -28
- package/src/config/capsule-serve.schema.json +0 -14
- package/src/config/index.d.ts +0 -52
- package/src/config/index.js +0 -22
- package/src/config/kernels/kernel-ref-digests.js +0 -348
- package/src/config/runtime/diagnostics/debug-logits.json +0 -54
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +0 -58
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +0 -58
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +0 -57
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +0 -33
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +0 -58
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +0 -58
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +0 -81
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +0 -213
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +0 -85
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +0 -81
- package/src/config/runtime/experiments/verify/gemma3-verify.json +0 -37
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +0 -36
- package/src/config/runtime/experiments/verify/lfm2-verify.json +0 -44
- package/src/config/runtime/experiments/verify/translategemma-verify.json +0 -37
- package/src/config/runtime/model/gemma2-debug.json +0 -78
- package/src/config/runtime/model/gemma2-pipeline-debug.json +0 -67
- package/src/config/runtime/model/gemma2-pipeline.json +0 -76
- package/src/config/runtime/model/gemma3-layer-probe.json +0 -86
- package/src/config/runtime/model/qwen3-5-layer-probe.json +0 -53
- package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +0 -109
- package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +0 -91
- package/src/config/runtime/optimization-candidates/registry.json +0 -5
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +0 -27
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +0 -43
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +0 -40
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +0 -46
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +0 -44
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +0 -61
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +0 -39
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +0 -42
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +0 -20
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +0 -27
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +0 -30
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +0 -62
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +0 -43
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +0 -64
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +0 -33
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +0 -17
- package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +0 -30
- package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +0 -110
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +0 -21
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +0 -53
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +0 -19
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +0 -53
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +0 -53
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +0 -86
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +0 -33
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +0 -17
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +0 -19
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +0 -47
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +0 -58
- package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +0 -87
- package/src/config/runtime/profiles/simulation.json +0 -29
- package/src/config/runtime/profiles/trace-layers.json +0 -125
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +0 -23
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +0 -30
- package/src/config/runtime/profiles/turboquant-contiguous.json +0 -30
- package/src/config/runtime/profiles/turboquant.json +0 -37
- package/src/config/runtime/profiles/verbose-trace.json +0 -37
- package/src/config/runtime/tiers/gemma4-128gb.json +0 -80
- package/src/config/runtime/tiers/gemma4-16gb.json +0 -83
- package/src/config/runtime/tiers/gemma4-24gb.json +0 -80
- package/src/config/runtime/tiers/gemma4-32gb.json +0 -80
- package/src/config/transforms/capability-transform-resolver.js +0 -4
- package/src/config/transforms/execution-graph/fusion.js +0 -436
- package/src/config/transforms/execution-graph/validation.js +0 -414
- package/src/config/transforms/execution-graph-transforms.js +0 -987
- package/src/converter/shard-packer.d.ts +0 -138
- package/src/converter/shard-packer.js +0 -425
- package/src/experimental/adapters/adapter-manager.d.ts +0 -200
- package/src/experimental/adapters/adapter-manager.js +0 -511
- package/src/experimental/adapters/adapter-registry.d.ts +0 -192
- package/src/experimental/adapters/adapter-registry.js +0 -477
- package/src/experimental/adapters/artifact-contract.d.ts +0 -83
- package/src/experimental/adapters/artifact-contract.js +0 -257
- package/src/experimental/adapters/index.d.ts +0 -114
- package/src/experimental/adapters/index.js +0 -54
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +0 -162
- package/src/experimental/bridge/trainer-artifact-bridge.js +0 -595
- package/src/experimental/browser/browser-converter.d.ts +0 -71
- package/src/experimental/browser/browser-converter.js +0 -978
- package/src/experimental/browser/file-picker.d.ts +0 -63
- package/src/experimental/browser/file-picker.js +0 -281
- package/src/experimental/browser/gguf-parser-browser.js +0 -17
- package/src/experimental/browser/quantization.js +0 -328
- package/src/experimental/browser/safetensors-parser-browser.js +0 -347
- package/src/experimental/browser/tensor-source-download.js +0 -245
- package/src/experimental/browser/tensor-source-file.d.ts +0 -26
- package/src/experimental/browser/tensor-source-file.js +0 -53
- package/src/experimental/browser/tensor-source-http.d.ts +0 -29
- package/src/experimental/browser/tensor-source-http.js +0 -130
- package/src/experimental/diffusion/image-regression.d.ts +0 -31
- package/src/experimental/diffusion/image-regression.js +0 -107
- package/src/experimental/diffusion/index.d.ts +0 -32
- package/src/experimental/diffusion/index.js +0 -18
- package/src/experimental/diffusion/pipeline.js +0 -727
- package/src/experimental/diffusion/scheduler.js +0 -153
- package/src/experimental/diffusion/text-encoder-gpu.js +0 -947
- package/src/experimental/diffusion/vae.js +0 -596
- package/src/experimental/distribution/descriptor-transport.d.ts +0 -117
- package/src/experimental/distribution/descriptor-transport.js +0 -342
- package/src/experimental/distribution/p2p-control-plane.d.ts +0 -52
- package/src/experimental/distribution/p2p-observability.d.ts +0 -116
- package/src/experimental/distribution/p2p-observability.js +0 -303
- package/src/experimental/distribution/p2p-webrtc-browser.d.ts +0 -37
- package/src/experimental/distribution/shard-delivery/transport.js +0 -831
- package/src/experimental/distribution/shard-delivery.d.ts +0 -251
- package/src/experimental/energy/index.d.ts +0 -2
- package/src/experimental/energy/index.js +0 -2
- package/src/experimental/energy/pipeline.js +0 -697
- package/src/experimental/energy/quintel.d.ts +0 -107
- package/src/experimental/energy/quintel.js +0 -218
- package/src/experimental/orchestration/energy-row-head-pipeline.js +0 -491
- package/src/experimental/orchestration/index.d.ts +0 -21
- package/src/experimental/orchestration/index.js +0 -12
- package/src/formats/quantization-constants.d.ts +0 -11
- package/src/formats/quantization-constants.js +0 -11
- package/src/formats/tokenizer/types.js +0 -22
- package/src/gpu/kernel-selector.d.ts +0 -11
- package/src/gpu/kernels/activation-static-qdq.d.ts +0 -22
- package/src/gpu/kernels/attention.d.ts +0 -253
- package/src/gpu/kernels/backward/adam.d.ts +0 -29
- package/src/gpu/kernels/backward/attention_backward.d.ts +0 -35
- package/src/gpu/kernels/backward/bias_add_backward.d.ts +0 -17
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +0 -24
- package/src/gpu/kernels/backward/conv2d_backward.d.ts +0 -31
- package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +0 -23
- package/src/gpu/kernels/backward/embed_backward.d.ts +0 -29
- package/src/gpu/kernels/backward/executor.d.ts +0 -34
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +0 -39
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +0 -32
- package/src/gpu/kernels/backward/gelu_backward.d.ts +0 -16
- package/src/gpu/kernels/backward/groupnorm_backward.d.ts +0 -24
- package/src/gpu/kernels/backward/index.d.ts +0 -26
- package/src/gpu/kernels/backward/layernorm_backward.d.ts +0 -22
- package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -31
- package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +0 -22
- package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +0 -24
- package/src/gpu/kernels/backward/rope_backward.d.ts +0 -25
- package/src/gpu/kernels/backward/scale_backward.d.ts +0 -16
- package/src/gpu/kernels/backward/silu_backward.d.ts +0 -16
- package/src/gpu/kernels/backward/softmax_backward.d.ts +0 -16
- package/src/gpu/kernels/backward/upsample2d_backward.d.ts +0 -21
- package/src/gpu/kernels/backward/utils.js +0 -2
- package/src/gpu/kernels/cast.d.ts +0 -67
- package/src/gpu/kernels/clamp.d.ts +0 -22
- package/src/gpu/kernels/conv2d.d.ts +0 -30
- package/src/gpu/kernels/cross_entropy_loss.d.ts +0 -21
- package/src/gpu/kernels/depthwise_conv1d.d.ts +0 -23
- package/src/gpu/kernels/depthwise_conv2d.d.ts +0 -29
- package/src/gpu/kernels/dequant.d.ts +0 -92
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +0 -19
- package/src/gpu/kernels/energy.d.ts +0 -113
- package/src/gpu/kernels/feature-check.d.ts +0 -42
- package/src/gpu/kernels/fused_ffn.d.ts +0 -97
- package/src/gpu/kernels/fused_matmul_residual.d.ts +0 -46
- package/src/gpu/kernels/fused_matmul_rmsnorm.d.ts +0 -64
- package/src/gpu/kernels/gather.d.ts +0 -144
- package/src/gpu/kernels/gelu.d.ts +0 -33
- package/src/gpu/kernels/gemma4-route-expert.d.ts +0 -35
- package/src/gpu/kernels/grouped_pointwise_conv2d.d.ts +0 -27
- package/src/gpu/kernels/groupnorm.d.ts +0 -27
- package/src/gpu/kernels/index.d.ts +0 -517
- package/src/gpu/kernels/index.js +0 -408
- package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +0 -9
- package/src/gpu/kernels/kernel-prewarm.d.ts +0 -3
- package/src/gpu/kernels/kv-cache-write.d.ts +0 -22
- package/src/gpu/kernels/kv-quantize.d.ts +0 -37
- package/src/gpu/kernels/layernorm.d.ts +0 -33
- package/src/gpu/kernels/linear_attention.d.ts +0 -27
- package/src/gpu/kernels/lm-head-argmax.d.ts +0 -28
- package/src/gpu/kernels/lm-head-select-logits.d.ts +0 -23
- package/src/gpu/kernels/logit-merge.d.ts +0 -86
- package/src/gpu/kernels/logit-merge.js +0 -266
- package/src/gpu/kernels/matmul.d.ts +0 -89
- package/src/gpu/kernels/modulate.d.ts +0 -29
- package/src/gpu/kernels/moe.d.ts +0 -113
- package/src/gpu/kernels/pixel_shuffle.d.ts +0 -27
- package/src/gpu/kernels/relu.d.ts +0 -18
- package/src/gpu/kernels/repeat_channels.d.ts +0 -21
- package/src/gpu/kernels/residual.d.ts +0 -65
- package/src/gpu/kernels/rmsnorm-pair.d.ts +0 -72
- package/src/gpu/kernels/rmsnorm-qk.d.ts +0 -41
- package/src/gpu/kernels/rmsnorm-stats.d.ts +0 -29
- package/src/gpu/kernels/rmsnorm.d.ts +0 -76
- package/src/gpu/kernels/rope-qk.d.ts +0 -41
- package/src/gpu/kernels/rope.d.ts +0 -48
- package/src/gpu/kernels/sample.d.ts +0 -70
- package/src/gpu/kernels/scale.d.ts +0 -40
- package/src/gpu/kernels/silu.d.ts +0 -77
- package/src/gpu/kernels/soft-embedding.d.ts +0 -28
- package/src/gpu/kernels/softmax.d.ts +0 -57
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +0 -44
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +0 -59
- package/src/gpu/kernels/split_qg.d.ts +0 -50
- package/src/gpu/kernels/split_qkv.d.ts +0 -53
- package/src/gpu/kernels/transpose.d.ts +0 -21
- package/src/gpu/kernels/types.d.ts +0 -21
- package/src/gpu/kernels/upsample2d.d.ts +0 -21
- package/src/gpu/kernels/utils.js +0 -9
- package/src/gpu/kernels/vision-average-pool.d.ts +0 -13
- package/src/gpu/kernels/vision-patch-embed.d.ts +0 -17
- package/src/gpu/kernels/vision-position-embedding.d.ts +0 -13
- package/src/gpu/kernels/vision-rope-2d.d.ts +0 -16
- package/src/gpu/kernels/vision-spatial-merge.d.ts +0 -19
- package/src/gpu/multi-model-recorder.d.ts +0 -21
- package/src/gpu/multi-model-recorder.js +0 -31
- package/src/gpu/partitioned-buffer-pool.d.ts +0 -28
- package/src/gpu/partitioned-buffer-pool.js +0 -57
- package/src/gpu/perf-guards.js +0 -71
- package/src/inference/browser-harness/benchmark-execution.js +0 -872
- package/src/inference/browser-harness/decode-diagnostics.js +0 -241
- package/src/inference/browser-harness/inference-execution.js +0 -543
- package/src/inference/browser-harness/model-initialization.js +0 -225
- package/src/inference/browser-harness/model-resolution.js +0 -137
- package/src/inference/browser-harness/report.js +0 -338
- package/src/inference/browser-harness/request.js +0 -263
- package/src/inference/browser-harness/runtime-config.d.ts +0 -25
- package/src/inference/browser-harness/runtime-config.js +0 -255
- package/src/inference/browser-harness/runtime-isolation.d.ts +0 -13
- package/src/inference/browser-harness/runtime-report.d.ts +0 -26
- package/src/inference/browser-harness/text-evidence.js +0 -694
- package/src/inference/browser-harness/text-execution.js +0 -757
- package/src/inference/browser-harness/text-input.js +0 -725
- package/src/inference/browser-harness/timing-diagnostics.js +0 -414
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -3
- package/src/inference/browser-harness-runtime-helpers.js +0 -3
- package/src/inference/browser-harness-text-helpers.js +0 -31
- package/src/inference/browser-harness.d.ts +0 -245
- package/src/inference/browser-harness.js +0 -337
- package/src/inference/expert-router.d.ts +0 -27
- package/src/inference/expert-router.js +0 -55
- package/src/inference/kv-cache/index.d.ts +0 -38
- package/src/inference/kv-cache/index.js +0 -22
- package/src/inference/kv-cache.d.ts +0 -39
- package/src/inference/kv-cache.js +0 -20
- package/src/inference/multi-model-network.d.ts +0 -139
- package/src/inference/multi-model-network.js +0 -435
- package/src/inference/multi-pipeline-pool.d.ts +0 -62
- package/src/inference/multi-pipeline-pool.js +0 -161
- package/src/inference/network-evolution.d.ts +0 -43
- package/src/inference/pipelines/diffusion/init.d.ts +0 -24
- package/src/inference/pipelines/diffusion/init.js +0 -138
- package/src/inference/pipelines/diffusion/profile-timing.js +0 -10
- package/src/inference/pipelines/diffusion/runtime-resources.js +0 -54
- package/src/inference/pipelines/diffusion/sd3/execution.js +0 -805
- package/src/inference/pipelines/diffusion/sd3/plan.js +0 -125
- package/src/inference/pipelines/diffusion/sd3-transformer.js +0 -343
- package/src/inference/pipelines/diffusion/sd3-weights.js +0 -287
- package/src/inference/pipelines/diffusion/text-encoder.js +0 -195
- package/src/inference/pipelines/diffusion/vae/execution.js +0 -803
- package/src/inference/pipelines/diffusion/weight-contract.js +0 -53
- package/src/inference/pipelines/diffusion/weights.js +0 -155
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +0 -27
- package/src/inference/pipelines/diffusion-gemma/config.js +0 -149
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +0 -31
- package/src/inference/pipelines/diffusion-gemma/index.js +0 -12
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +0 -69
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +0 -380
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +0 -128
- package/src/inference/pipelines/diffusion-gemma/sampling.js +0 -395
- package/src/inference/pipelines/rng.js +0 -17
- package/src/inference/pipelines/structured/index.d.ts +0 -10
- package/src/inference/pipelines/structured/index.js +0 -10
- package/src/inference/pipelines/text/attention/heterogeneous-contract.js +0 -8
- package/src/inference/pipelines/text/attention/index.js +0 -24
- package/src/inference/pipelines/text/attention/record.js +0 -2
- package/src/inference/pipelines/text/attention/run.js +0 -2
- package/src/inference/pipelines/text/config/validation.js +0 -9
- package/src/inference/pipelines/text/debug-utils/index.js +0 -39
- package/src/inference/pipelines/text/debug-utils/profiles.js +0 -8
- package/src/inference/pipelines/text/debug-utils.js +0 -34
- package/src/inference/pipelines/text/ffn/index.js +0 -16
- package/src/inference/pipelines/text/model-load.d.ts +0 -59
- package/src/inference/test-harness.d.ts +0 -172
- package/src/inference/test-harness.js +0 -470
- package/src/loader/dtype-utils.d.ts +0 -40
- package/src/loader/embedding-loader.d.ts +0 -66
- package/src/loader/index.d.ts +0 -152
- package/src/loader/index.js +0 -126
- package/src/loader/loader-state.d.ts +0 -51
- package/src/loader/loader-state.js +0 -142
- package/src/loader/manifest-config.d.ts +0 -118
- package/src/loader/multi-model-loader.d.ts +0 -59
- package/src/loader/multi-model-loader.js +0 -141
- package/src/loader/quantization-constants.d.ts +0 -11
- package/src/loader/quantization-constants.js +0 -13
- package/src/loader/shard-resolver.d.ts +0 -13
- package/src/loader/tensors/tensor-loader.d.ts +0 -197
- package/src/loader/tensors/tensor-reader.d.ts +0 -33
- package/src/loader/tensors/tensor-role.d.ts +0 -7
- package/src/loader/weight-downcast.d.ts +0 -62
- package/src/loader/weights.js +0 -4
- package/src/loaders/index.d.ts +0 -4
- package/src/loaders/index.js +0 -4
- package/src/models/diffusiongemma.d.ts +0 -7
- package/src/models/diffusiongemma.js +0 -25
- package/src/models/embeddinggemma.d.ts +0 -7
- package/src/models/embeddinggemma.js +0 -24
- package/src/models/family.d.ts +0 -24
- package/src/models/family.js +0 -25
- package/src/models/gemma3.d.ts +0 -7
- package/src/models/gemma3.js +0 -33
- package/src/models/gemma4.d.ts +0 -7
- package/src/models/gemma4.js +0 -65
- package/src/models/qwen3.d.ts +0 -7
- package/src/models/qwen3.js +0 -36
- package/src/rules/capability-transform-resolver.js +0 -80
- package/src/tooling/capsule-v2.d.ts +0 -11
- package/src/tooling/model-capsule-forge.js +0 -478
- package/src/tooling/source-runtime/bundle-materialize.js +0 -298
- package/src/tooling-experimental-exports.browser.js +0 -2
- package/src/tooling-experimental-exports.d.ts +0 -7
- package/src/tooling-experimental-exports.js +0 -2
- package/src/tooling-experimental-exports.shared.d.ts +0 -61
- package/src/tooling-experimental-exports.shared.js +0 -57
- /package/src/{config/transforms → converter}/execution-graph/dtype.js +0 -0
- /package/src/{tooling/source-runtime/bundle-plan.js → formats/source-runtime.js} +0 -0
- /package/src/{inference → tooling}/browser-harness/benchmark-rerank.js +0 -0
- /package/src/{inference → tooling}/browser-harness/diffusion-performance.js +0 -0
- /package/src/{inference → tooling}/browser-harness/runtime-isolation.js +0 -0
- /package/src/{inference → tooling}/browser-harness/runtime-report.js +0 -0
- /package/src/{inference → tooling}/browser-harness/suite-summary.js +0 -0
- /package/src/{inference → tooling}/browser-harness-contract-helpers.js +0 -0
- /package/src/{inference → tooling}/browser-harness-diffusion-energy-suites.js +0 -0
- /package/src/{inference → tooling}/browser-harness-report-helpers.js +0 -0
|
@@ -0,0 +1,927 @@
|
|
|
1
|
+
import { widenAttentionKvToF32 } from './execution-graph/kv-storage.js';
|
|
2
|
+
export { widenAttentionKvToF32 } from './execution-graph/kv-storage.js';
|
|
3
|
+
import { F16_TO_F32_ACTIVATION_MAP, F32_TO_F16_ACTIVATION_MAP, KERNEL_FILE_PRECISION_PATCHES, cloneGraph, deriveKernelEntry, deriveKernelKey, deriveLinearDecodeF16KernelEntry, narrowToF16Activations, useQwen36F16Activations, useQwenDecodeF16Matmuls, useQwenF16PrimaryMatmuls } from './execution-graph/dtype.js';
|
|
4
|
+
import { LAYER_PROJECTION_OPS, useGemma4Int4PleAf16Activations, useGemma4Int4PleSelectiveF16Decode, useGemma4TextF16ActivationsForLane } from './execution-graph/fusion.js';
|
|
5
|
+
import { ATTENTION_F16KV_TO_F32_ENTRIES, deriveMappedKernelEntry, deriveQ4DecodeF32ActivationKernelEntry, deriveQ4PrefillF32ActivationKernelEntry, findKernelKeyByFileAndEntry, hasKernelFile, remapDenseQ4KPrefillToQ4Native, remapStepKeys, removeSubgroups, widenProjectionWeightsToF32, widenToF32Activations } from './execution-graph/validation.js';
|
|
6
|
+
export { remapDenseQ4KPrefillToQ4Native, removeSubgroups, widenProjectionWeightsToF32, widenToF32Activations } from './execution-graph/validation.js';
|
|
7
|
+
export { useGemma4Int4PleAf16Activations, useGemma4Int4PleSelectiveF16Decode } from './execution-graph/fusion.js';
|
|
8
|
+
export { narrowToF16Activations, useQwen36F16Activations, useQwenDecodeF16Matmuls, useQwenF16PrimaryMatmuls } from './execution-graph/dtype.js';// =============================================================================
|
|
9
|
+
// Execution Graph Transforms
|
|
10
|
+
// =============================================================================
|
|
11
|
+
//
|
|
12
|
+
// Pure functions that take an execution-v1 graph (as stamped in the manifest)
|
|
13
|
+
// and return a modified copy. Replaces the kernel path registry system.
|
|
14
|
+
//
|
|
15
|
+
// Each transform: (graph, ctx) => newGraph | null
|
|
16
|
+
// Returns null if not applicable (no-op).
|
|
17
|
+
// Must be pure — no mutation, return new objects.
|
|
18
|
+
|
|
19
|
+
// =============================================================================
|
|
20
|
+
// Helpers
|
|
21
|
+
// =============================================================================
|
|
22
|
+
|
|
23
|
+
/*
|
|
24
|
+
* Deep-clone an execution graph.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
/*
|
|
28
|
+
* Shader files that require subgroups even though "subgroup" is not in the filename.
|
|
29
|
+
* Online attention kernels use subgroup reductions internally.
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
export function getKernelFilePrecisionPatch(kernel) {
|
|
33
|
+
return KERNEL_FILE_PRECISION_PATCHES.get(kernel) ?? null;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/*
|
|
37
|
+
* Find the first kernel key used by matching ops in a phase whose shader file
|
|
38
|
+
* satisfies the provided predicate.
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
/*
|
|
42
|
+
* Find an existing kernel key by shader file and entry point.
|
|
43
|
+
*/
|
|
44
|
+
|
|
45
|
+
function normalizeLayerType(layerType) {
|
|
46
|
+
return typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function isLinearAttentionLayerType(layerType) {
|
|
50
|
+
const normalized = normalizeLayerType(layerType);
|
|
51
|
+
return normalized === 'linear_attention'
|
|
52
|
+
|| normalized === 'linear'
|
|
53
|
+
|| normalized === 'gated_delta'
|
|
54
|
+
|| normalized === 'gated_delta_net';
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function buildGroupedLayerEntries(baseStep, targetLayers, replacementKernelKey) {
|
|
58
|
+
const groupedEntries = [];
|
|
59
|
+
if (!Array.isArray(baseStep) || baseStep.length < 2) {
|
|
60
|
+
return groupedEntries;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const totalLayers = targetLayers.allLayers;
|
|
64
|
+
const targeted = targetLayers.matchingLayers;
|
|
65
|
+
const remaining = totalLayers.filter((layerIdx) => !targeted.includes(layerIdx));
|
|
66
|
+
|
|
67
|
+
if (remaining.length > 0) {
|
|
68
|
+
groupedEntries.push({
|
|
69
|
+
layers: remaining,
|
|
70
|
+
steps: [baseStep],
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
if (targeted.length > 0) {
|
|
74
|
+
const replacement = [...baseStep];
|
|
75
|
+
replacement[1] = replacementKernelKey;
|
|
76
|
+
groupedEntries.push({
|
|
77
|
+
layers: targeted,
|
|
78
|
+
steps: [replacement],
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
return groupedEntries;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// =============================================================================
|
|
86
|
+
// Transform: removeSubgroups
|
|
87
|
+
// =============================================================================
|
|
88
|
+
|
|
89
|
+
/*
|
|
90
|
+
* Remove subgroup shader dependencies from decode and postLayer steps.
|
|
91
|
+
* Prefill steps are left untouched (they already use tiled matmul).
|
|
92
|
+
*
|
|
93
|
+
* Returns null if the graph has no subgroup kernels.
|
|
94
|
+
*
|
|
95
|
+
*/
|
|
96
|
+
|
|
97
|
+
// =============================================================================
|
|
98
|
+
// Transform: widenToF32Activations
|
|
99
|
+
// =============================================================================
|
|
100
|
+
|
|
101
|
+
/*
|
|
102
|
+
* Activation-only widening: f16-activation shaders → f32-activation variants
|
|
103
|
+
* that still use f16 for weights and KV cache. Requires shader-f16 for weight
|
|
104
|
+
* and KV buffer reads.
|
|
105
|
+
*/
|
|
106
|
+
|
|
107
|
+
export function resolveF16ToF32ActivationKernel(kernel) {
|
|
108
|
+
return F16_TO_F32_ACTIVATION_MAP.get(kernel) ?? null;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/*
|
|
112
|
+
* Activation-only narrowing: f32-activation shaders that still consume f16
|
|
113
|
+
* weights/KV are rewritten onto the matching f16-activation lane.
|
|
114
|
+
*
|
|
115
|
+
* This is the inverse of `F16_TO_F32_ACTIVATION_MAP` and is used when a
|
|
116
|
+
* runtime session explicitly requests f16 activations for an execution-v1
|
|
117
|
+
* graph that was authored with conservative f32 activation defaults.
|
|
118
|
+
*/
|
|
119
|
+
|
|
120
|
+
F32_TO_F16_ACTIVATION_MAP.set('gather.wgsl', 'gather_f16.wgsl');
|
|
121
|
+
F32_TO_F16_ACTIVATION_MAP.set('attention_head256_f16kv.wgsl', 'attention_small_f16.wgsl');
|
|
122
|
+
// head512 pure-f16 prefill is model-scoped below; keep generic Gemma 4 E2B
|
|
123
|
+
// f16 requests fail-closed until that path has its own evidence.
|
|
124
|
+
F32_TO_F16_ACTIVATION_MAP.delete('attention_head512_f16kv.wgsl');
|
|
125
|
+
|
|
126
|
+
/*
|
|
127
|
+
* Correctness fallback: preserve f16 weights where possible, but widen both
|
|
128
|
+
* activations and KV-cache interactions onto the stable f32 execution lane.
|
|
129
|
+
* Used for alternate-plan recovery after finiteness failure.
|
|
130
|
+
*/
|
|
131
|
+
const F16_TO_F32_CORRECTNESS_FALLBACK_MAP = new Map([
|
|
132
|
+
['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
|
|
133
|
+
['rope_f16.wgsl', 'rope.wgsl'],
|
|
134
|
+
['residual_f16.wgsl', 'residual.wgsl'],
|
|
135
|
+
['gelu_f16.wgsl', 'gelu.wgsl'],
|
|
136
|
+
['silu_f16.wgsl', 'silu.wgsl'],
|
|
137
|
+
['sample_f16.wgsl', 'sample.wgsl'],
|
|
138
|
+
['gather_f16.wgsl', 'gather.wgsl'],
|
|
139
|
+
['gather_f16_f16_out.wgsl', 'gather.wgsl'],
|
|
140
|
+
['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
|
|
141
|
+
['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
|
|
142
|
+
['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
|
|
143
|
+
['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
|
|
144
|
+
['attention_decode_online_f16.wgsl', 'attention_streaming.wgsl'],
|
|
145
|
+
['attention_decode_chunked_f16.wgsl', 'attention_streaming.wgsl'],
|
|
146
|
+
['attention_small_f16.wgsl', 'attention_small.wgsl'],
|
|
147
|
+
['attention_streaming_f16.wgsl', 'attention_streaming.wgsl'],
|
|
148
|
+
...ATTENTION_F16KV_TO_F32_ENTRIES,
|
|
149
|
+
]);
|
|
150
|
+
|
|
151
|
+
/*
|
|
152
|
+
* Full f32 widening: every shader that uses `enable f16;` is replaced with a
|
|
153
|
+
* pure-f32 equivalent. Used when the GPU cannot compile any f16 WGSL at all.
|
|
154
|
+
* Covers f16-activation, f16-weight (f16w), and f16-KV (f16kv) kernels.
|
|
155
|
+
*/
|
|
156
|
+
|
|
157
|
+
/*
|
|
158
|
+
* Widen all f16-activation shaders to f32-activation equivalents.
|
|
159
|
+
*
|
|
160
|
+
* Returns null if the graph contains fused_ffn_f16.wgsl (no direct f32
|
|
161
|
+
* equivalent exists) or if no f16 activation shaders are present.
|
|
162
|
+
*
|
|
163
|
+
* NOTE: The caller is responsible for also updating session.activationDtype
|
|
164
|
+
* to reflect the widened dtype.
|
|
165
|
+
*
|
|
166
|
+
*/
|
|
167
|
+
|
|
168
|
+
/*
|
|
169
|
+
* Widen an f16 execution graph onto the stable f32 correctness lane used for
|
|
170
|
+
* alternate-plan recovery after finiteness failure.
|
|
171
|
+
*
|
|
172
|
+
*/
|
|
173
|
+
export function widenToF32CorrectnessFallback(graph, ctx) {
|
|
174
|
+
if (hasKernelFile(graph, 'fused_ffn_f16.wgsl')) {
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const hasTargetShader = Object.values(graph.kernels).some(
|
|
179
|
+
(entry) => F16_TO_F32_CORRECTNESS_FALLBACK_MAP.has(entry.kernel)
|
|
180
|
+
);
|
|
181
|
+
if (!hasTargetShader) {
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
const result = cloneGraph(graph);
|
|
186
|
+
for (const [key, entry] of Object.entries(result.kernels)) {
|
|
187
|
+
const q4FallbackEntry = deriveQ4DecodeF32ActivationKernelEntry(entry)
|
|
188
|
+
?? deriveQ4PrefillF32ActivationKernelEntry(entry);
|
|
189
|
+
if (q4FallbackEntry) {
|
|
190
|
+
result.kernels[key] = q4FallbackEntry;
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
const replacement = F16_TO_F32_CORRECTNESS_FALLBACK_MAP.get(entry.kernel);
|
|
194
|
+
if (replacement !== undefined) {
|
|
195
|
+
result.kernels[key] = deriveMappedKernelEntry(entry, replacement);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
return result;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/*
|
|
202
|
+
* Narrow f32-activation shaders back onto their f16-activation equivalents.
|
|
203
|
+
*
|
|
204
|
+
* Returns null if the graph has no supported f32-activation kernels to swap or
|
|
205
|
+
* if the runtime did not explicitly request f16 activations on an f16-capable
|
|
206
|
+
* GPU.
|
|
207
|
+
*
|
|
208
|
+
*/
|
|
209
|
+
|
|
210
|
+
// =============================================================================
|
|
211
|
+
// Transform: swapPrefillAttention
|
|
212
|
+
// =============================================================================
|
|
213
|
+
|
|
214
|
+
const PREFILL_ATTENTION_PAIRS = new Map([
|
|
215
|
+
['attention_streaming_f16kv.wgsl', 'attention_small_f16kv.wgsl'],
|
|
216
|
+
['attention_small_f16kv.wgsl', 'attention_streaming_f16kv.wgsl'],
|
|
217
|
+
['attention_streaming_f16.wgsl', 'attention_small_f16.wgsl'],
|
|
218
|
+
['attention_small_f16.wgsl', 'attention_streaming_f16.wgsl'],
|
|
219
|
+
]);
|
|
220
|
+
|
|
221
|
+
function graphUsesKernelKeyInPrefill(graph, kernelKey) {
|
|
222
|
+
for (const entry of graph.prefill || []) {
|
|
223
|
+
if (Array.isArray(entry)) {
|
|
224
|
+
if (entry[1] === kernelKey) {
|
|
225
|
+
return true;
|
|
226
|
+
}
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
for (const step of entry.steps) {
|
|
233
|
+
if (Array.isArray(step) && step[1] === kernelKey) {
|
|
234
|
+
return true;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
return false;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/*
|
|
242
|
+
* Swap prefill attention kernel between streaming and small variants.
|
|
243
|
+
*
|
|
244
|
+
* The `opts` parameter specifies the direction:
|
|
245
|
+
* { from: 'attention_streaming_f16kv.wgsl', to: 'attention_small_f16kv.wgsl' }
|
|
246
|
+
*
|
|
247
|
+
* If `from`/`to` are not provided, uses the bidirectional pair map.
|
|
248
|
+
* Returns null if no matching prefill attention kernel is found.
|
|
249
|
+
*
|
|
250
|
+
*/
|
|
251
|
+
export function swapPrefillAttention(graph, ctx, opts) {
|
|
252
|
+
const from = opts?.from;
|
|
253
|
+
const to = opts?.to;
|
|
254
|
+
|
|
255
|
+
const result = cloneGraph(graph);
|
|
256
|
+
let changed = false;
|
|
257
|
+
|
|
258
|
+
for (const [key, entry] of Object.entries(result.kernels)) {
|
|
259
|
+
let target;
|
|
260
|
+
|
|
261
|
+
if (from && to) {
|
|
262
|
+
// Explicit direction: only swap if the kernel matches `from`
|
|
263
|
+
if (entry.kernel === from) {
|
|
264
|
+
target = to;
|
|
265
|
+
}
|
|
266
|
+
} else {
|
|
267
|
+
// Bidirectional: use pair map
|
|
268
|
+
target = PREFILL_ATTENTION_PAIRS.get(entry.kernel);
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
if (target !== undefined) {
|
|
272
|
+
const usedInPrefill = graphUsesKernelKeyInPrefill(graph, key);
|
|
273
|
+
if (usedInPrefill) {
|
|
274
|
+
result.kernels[key] = deriveKernelEntry(entry, target, entry.entry);
|
|
275
|
+
changed = true;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
return changed ? result : null;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
// =============================================================================
|
|
284
|
+
// Transform: useHead256PrefillAttention
|
|
285
|
+
// =============================================================================
|
|
286
|
+
|
|
287
|
+
/*
|
|
288
|
+
* Promote small-tile prefill attention onto the fixed 256-dim shared-block kernel.
|
|
289
|
+
*
|
|
290
|
+
*/
|
|
291
|
+
export function useHead256SmallPrefillAttention(graph, ctx) {
|
|
292
|
+
return swapPrefillAttention(graph, ctx, {
|
|
293
|
+
from: 'attention_small_f16kv.wgsl',
|
|
294
|
+
to: 'attention_head256_f16kv.wgsl',
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/*
|
|
299
|
+
* Promote prefill attention onto the fixed 256-dim shared-block kernel.
|
|
300
|
+
*
|
|
301
|
+
*/
|
|
302
|
+
export function useHead256PrefillAttention(graph, ctx) {
|
|
303
|
+
let current = graph;
|
|
304
|
+
let changed = false;
|
|
305
|
+
|
|
306
|
+
const smallRemap = useHead256SmallPrefillAttention(current, ctx);
|
|
307
|
+
if (smallRemap) {
|
|
308
|
+
current = smallRemap;
|
|
309
|
+
changed = true;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
const streamingRemap = swapPrefillAttention(current, ctx, {
|
|
313
|
+
from: 'attention_streaming_f16kv.wgsl',
|
|
314
|
+
to: 'attention_head256_f16kv.wgsl',
|
|
315
|
+
});
|
|
316
|
+
if (streamingRemap) {
|
|
317
|
+
current = streamingRemap;
|
|
318
|
+
changed = true;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
return changed ? current : null;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// =============================================================================
|
|
325
|
+
// Transform: widenProjectionWeightsToF32
|
|
326
|
+
// =============================================================================
|
|
327
|
+
|
|
328
|
+
/*
|
|
329
|
+
* Known layer projection ops. Only these are widened; lm_head and embed are
|
|
330
|
+
* excluded.
|
|
331
|
+
*/
|
|
332
|
+
|
|
333
|
+
function resolveDensePrefillProjectionKernel(ctx) {
|
|
334
|
+
return ctx.activationDtype === 'f16'
|
|
335
|
+
? 'matmul_f16.wgsl'
|
|
336
|
+
: 'matmul_f16w_f32a.wgsl';
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/*
|
|
340
|
+
* Replace projection matmul kernels with f32 weight variants.
|
|
341
|
+
*
|
|
342
|
+
* Applies only to layer projection steps (q/k/v/o/gate/up/down), NOT lm_head
|
|
343
|
+
* or embed.
|
|
344
|
+
*
|
|
345
|
+
* Returns null if no applicable projection kernels are found.
|
|
346
|
+
*
|
|
347
|
+
*/
|
|
348
|
+
|
|
349
|
+
// =============================================================================
|
|
350
|
+
// Transform: remapDenseQ4KPrefillToQ4Native
|
|
351
|
+
// =============================================================================
|
|
352
|
+
|
|
353
|
+
/*
|
|
354
|
+
* Replace dense prefill projection kernels with Q4-native prefill variants.
|
|
355
|
+
*
|
|
356
|
+
* This applies only when the graph already exposes a compatible fused Q4 decode
|
|
357
|
+
* projection kernel. All prefill layer projections are remapped to the shared-A
|
|
358
|
+
* batched multicol Q4 prefill kernel so the transformed path remains valid for
|
|
359
|
+
* `M > 1` prefill workloads.
|
|
360
|
+
*
|
|
361
|
+
* Returns null when the graph does not have the required dense-prefill + Q4
|
|
362
|
+
* decode shape.
|
|
363
|
+
*
|
|
364
|
+
*/
|
|
365
|
+
|
|
366
|
+
// =============================================================================
|
|
367
|
+
// Transform: remapQ4KPrefillToDense
|
|
368
|
+
// =============================================================================
|
|
369
|
+
|
|
370
|
+
/*
|
|
371
|
+
* Replace fused Q4K prefill projection kernels with dense tiled variants.
|
|
372
|
+
*
|
|
373
|
+
* Decode remains unchanged so the runtime can keep using fused Q4K decode while
|
|
374
|
+
* the loader exposes mixed dense+Q4K materializations for prefill.
|
|
375
|
+
*
|
|
376
|
+
* Returns null when the graph has no fused Q4K prefill projection kernels.
|
|
377
|
+
*
|
|
378
|
+
*/
|
|
379
|
+
export function remapQ4KPrefillToDense(graph, ctx) {
|
|
380
|
+
const q4PrefillProjectionSteps = (graph.prefill || []).filter((step) => {
|
|
381
|
+
if (!LAYER_PROJECTION_OPS.has(step[0])) {
|
|
382
|
+
return false;
|
|
383
|
+
}
|
|
384
|
+
const entry = graph.kernels[step[1]];
|
|
385
|
+
return entry != null && entry.kernel.startsWith('fused_matmul_q4');
|
|
386
|
+
});
|
|
387
|
+
if (q4PrefillProjectionSteps.length === 0) {
|
|
388
|
+
return null;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
const denseKernelFile = resolveDensePrefillProjectionKernel(ctx);
|
|
392
|
+
const result = cloneGraph(graph);
|
|
393
|
+
let denseKey = findKernelKeyByFileAndEntry(result, denseKernelFile, 'main');
|
|
394
|
+
if (!denseKey) {
|
|
395
|
+
const sourceKey = q4PrefillProjectionSteps[0][1];
|
|
396
|
+
const sourceEntry = result.kernels[sourceKey];
|
|
397
|
+
denseKey = deriveKernelKey(result.kernels, sourceKey, '_prefill_dense');
|
|
398
|
+
result.kernels[denseKey] = deriveKernelEntry(
|
|
399
|
+
sourceEntry,
|
|
400
|
+
denseKernelFile,
|
|
401
|
+
'main',
|
|
402
|
+
null
|
|
403
|
+
);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
let changed = false;
|
|
407
|
+
result.prefill = (result.prefill || []).map((step) => {
|
|
408
|
+
if (!LAYER_PROJECTION_OPS.has(step[0])) {
|
|
409
|
+
return step;
|
|
410
|
+
}
|
|
411
|
+
const entry = result.kernels[step[1]];
|
|
412
|
+
if (!entry || !entry.kernel.startsWith('fused_matmul_q4')) {
|
|
413
|
+
return step;
|
|
414
|
+
}
|
|
415
|
+
if (step[1] === denseKey) {
|
|
416
|
+
return step;
|
|
417
|
+
}
|
|
418
|
+
changed = true;
|
|
419
|
+
const next = [...step];
|
|
420
|
+
next[1] = denseKey;
|
|
421
|
+
return next;
|
|
422
|
+
});
|
|
423
|
+
|
|
424
|
+
return changed ? result : null;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// =============================================================================
|
|
428
|
+
// Transform: useLinearDecodeProjectionF16
|
|
429
|
+
// =============================================================================
|
|
430
|
+
|
|
431
|
+
/*
|
|
432
|
+
* Remap the linear-attention q_proj decode step onto the f16-activation fused
|
|
433
|
+
* Q4 kernel for linear-attention layers only. Full-attention layers keep the
|
|
434
|
+
* manifest-wide f32 activation contract.
|
|
435
|
+
*
|
|
436
|
+
* Only q_proj is remapped. o_proj is intentionally excluded: the o_proj
|
|
437
|
+
* output enters the residual stream directly, and f16 truncation there
|
|
438
|
+
* accumulates across the 18 linear-attention layers in the Qwen 3.5 pattern,
|
|
439
|
+
* corrupting the logit distribution (empirically verified: degenerate
|
|
440
|
+
* repetitive output under greedy decode). q_proj f16 is safe because the
|
|
441
|
+
* linear attention core absorbs the f16 input into its f32 internal state.
|
|
442
|
+
*
|
|
443
|
+
*/
|
|
444
|
+
export function useLinearDecodeProjectionF16(graph, ctx) {
|
|
445
|
+
const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
|
|
446
|
+
if (!layerTypes || layerTypes.length === 0) {
|
|
447
|
+
return null;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
const matchingLayers = layerTypes
|
|
451
|
+
.map((layerType, layerIdx) => ({ layerType, layerIdx }))
|
|
452
|
+
.filter(({ layerType }) => isLinearAttentionLayerType(layerType))
|
|
453
|
+
.map(({ layerIdx }) => layerIdx);
|
|
454
|
+
if (matchingLayers.length === 0) {
|
|
455
|
+
return null;
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
const result = cloneGraph(graph);
|
|
459
|
+
const targetLayers = {
|
|
460
|
+
allLayers: layerTypes.map((_, layerIdx) => layerIdx),
|
|
461
|
+
matchingLayers,
|
|
462
|
+
};
|
|
463
|
+
const qProjIndex = (result.decode || []).findIndex((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
|
|
464
|
+
if (qProjIndex === -1) {
|
|
465
|
+
return null;
|
|
466
|
+
}
|
|
467
|
+
const qProjStep = result.decode[qProjIndex];
|
|
468
|
+
const qProjKernelKey = qProjStep[1];
|
|
469
|
+
const qProjKernel = result.kernels[qProjKernelKey];
|
|
470
|
+
if (!qProjKernel) {
|
|
471
|
+
return null;
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
const derivedEntry = deriveLinearDecodeF16KernelEntry(qProjKernel);
|
|
475
|
+
if (!derivedEntry) {
|
|
476
|
+
return null;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
const derivedKey = deriveKernelKey(result.kernels, qProjKernelKey, '_linear_f16out');
|
|
480
|
+
result.kernels[derivedKey] = derivedEntry;
|
|
481
|
+
const groupedEntries = buildGroupedLayerEntries(qProjStep, targetLayers, derivedKey);
|
|
482
|
+
if (groupedEntries.length === 0) {
|
|
483
|
+
return null;
|
|
484
|
+
}
|
|
485
|
+
result.decode = [
|
|
486
|
+
...result.decode.slice(0, qProjIndex),
|
|
487
|
+
...groupedEntries,
|
|
488
|
+
...result.decode.slice(qProjIndex + 1),
|
|
489
|
+
];
|
|
490
|
+
|
|
491
|
+
return result;
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
// =============================================================================
|
|
495
|
+
// Transform: remapQ4KDecodeToGemv
|
|
496
|
+
// =============================================================================
|
|
497
|
+
|
|
498
|
+
/*
|
|
499
|
+
* Replace fused Q4K decode projection kernels with GEMV subgroup variants.
|
|
500
|
+
*
|
|
501
|
+
* When Q4K weights have f16 materializations (mixed/dense loader mode), the
|
|
502
|
+
* GEMV subgroup kernel on pre-dequantized f16 weights is significantly faster
|
|
503
|
+
* than the fused Q4K kernel for M=1 decode (empirically 2.3x on Apple M-series).
|
|
504
|
+
*
|
|
505
|
+
* After this transform no decode kernels reference fused_matmul_q4*, which
|
|
506
|
+
* signals the loader to use dense materialization (f16 only — no Q4K buffer
|
|
507
|
+
* retained in GPU memory, reducing peak VRAM).
|
|
508
|
+
*
|
|
509
|
+
* Only layer projection ops are remapped. Non-matmul ops (rmsnorm, rope,
|
|
510
|
+
* attention, residual, activation) are left untouched.
|
|
511
|
+
*
|
|
512
|
+
*/
|
|
513
|
+
export function remapQ4KDecodeToGemv(graph, ctx) {
|
|
514
|
+
if (ctx.activationDtype === 'f16') {
|
|
515
|
+
return null;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
const decodeSteps = graph.decode || [];
|
|
519
|
+
const fusedDecodeKeys = new Set();
|
|
520
|
+
for (const step of decodeSteps) {
|
|
521
|
+
if (!Array.isArray(step)) continue;
|
|
522
|
+
const kernelKey = step[1];
|
|
523
|
+
const entry = graph.kernels[kernelKey];
|
|
524
|
+
if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
|
|
525
|
+
fusedDecodeKeys.add(kernelKey);
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
if (fusedDecodeKeys.size === 0) {
|
|
529
|
+
return null;
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
const result = cloneGraph(graph);
|
|
533
|
+
const keyMap = new Map();
|
|
534
|
+
|
|
535
|
+
for (const key of fusedDecodeKeys) {
|
|
536
|
+
const newKey = deriveKernelKey(result.kernels, key, '_gemv');
|
|
537
|
+
result.kernels[newKey] = deriveKernelEntry(
|
|
538
|
+
result.kernels[key],
|
|
539
|
+
'matmul_gemv_subgroup.wgsl',
|
|
540
|
+
'main_multicol',
|
|
541
|
+
null
|
|
542
|
+
);
|
|
543
|
+
keyMap.set(key, newKey);
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
result.decode = remapStepKeys(result.decode, keyMap);
|
|
547
|
+
return result;
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
// =============================================================================
|
|
551
|
+
// Transform: remapQ4KDecodeAttentionToGemv (diagnostic)
|
|
552
|
+
// =============================================================================
|
|
553
|
+
|
|
554
|
+
const ATTENTION_PROJECTION_OPS = new Set(['q_proj', 'k_proj', 'v_proj', 'o_proj']);
|
|
555
|
+
|
|
556
|
+
/*
|
|
557
|
+
* Replace fused Q4K ATTENTION-ONLY decode projection kernels with GEMV
|
|
558
|
+
* subgroup variants, leaving FFN projections (gate/up/down_proj) untouched.
|
|
559
|
+
*
|
|
560
|
+
* Diagnostic transform for isolating whether the GEMV correctness regression
|
|
561
|
+
* originates in the attention or FFN projection path. Because FFN ops keep
|
|
562
|
+
* their fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight
|
|
563
|
+
* loader remains in mixed-materialization mode.
|
|
564
|
+
*
|
|
565
|
+
*/
|
|
566
|
+
export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
|
|
567
|
+
if (ctx.activationDtype === 'f16') {
|
|
568
|
+
return null;
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
const decodeSteps = graph.decode || [];
|
|
572
|
+
const attnFusedKeys = new Set();
|
|
573
|
+
for (const step of decodeSteps) {
|
|
574
|
+
if (!Array.isArray(step)) continue;
|
|
575
|
+
const op = step[0];
|
|
576
|
+
if (!ATTENTION_PROJECTION_OPS.has(op)) continue;
|
|
577
|
+
const kernelKey = step[1];
|
|
578
|
+
const entry = graph.kernels[kernelKey];
|
|
579
|
+
if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
|
|
580
|
+
attnFusedKeys.add(kernelKey);
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
if (attnFusedKeys.size === 0) {
|
|
584
|
+
return null;
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
const result = cloneGraph(graph);
|
|
588
|
+
const keyMap = new Map();
|
|
589
|
+
|
|
590
|
+
for (const key of attnFusedKeys) {
|
|
591
|
+
const newKey = deriveKernelKey(result.kernels, key, '_attn_gemv');
|
|
592
|
+
result.kernels[newKey] = deriveKernelEntry(
|
|
593
|
+
result.kernels[key],
|
|
594
|
+
'matmul_gemv_subgroup.wgsl',
|
|
595
|
+
'main_multicol',
|
|
596
|
+
null
|
|
597
|
+
);
|
|
598
|
+
keyMap.set(key, newKey);
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
// Only remap attention projection steps, leave FFN steps unchanged.
|
|
602
|
+
result.decode = result.decode.map((step) => {
|
|
603
|
+
if (!Array.isArray(step)) return step;
|
|
604
|
+
if (!ATTENTION_PROJECTION_OPS.has(step[0])) return step;
|
|
605
|
+
const replacement = keyMap.get(step[1]);
|
|
606
|
+
if (replacement !== undefined) {
|
|
607
|
+
const newStep = [...step];
|
|
608
|
+
newStep[1] = replacement;
|
|
609
|
+
return newStep;
|
|
610
|
+
}
|
|
611
|
+
return step;
|
|
612
|
+
});
|
|
613
|
+
|
|
614
|
+
return result;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
// =============================================================================
|
|
618
|
+
// Transform: remapQ4KDecodeAttentionToFusedQ4KGemv
|
|
619
|
+
// =============================================================================
|
|
620
|
+
|
|
621
|
+
/*
|
|
622
|
+
* Replace fused Q4K ATTENTION-ONLY decode projection kernels with the
|
|
623
|
+
* optimised fused Q4K GEMV variant (main_gemv), which combines shared-A
|
|
624
|
+
* cooperative loading with fast nibble extraction for maximum M=1 throughput
|
|
625
|
+
* while preserving full Q4K dequant precision (no f16 weight materialization).
|
|
626
|
+
*
|
|
627
|
+
* This is the production fix for the f16-precision-loss regression observed
|
|
628
|
+
* when attention projections use the f16-weight GEMV path: softmax amplifies
|
|
629
|
+
* the f16 round-trip error in Q/K/V projections, causing garbage output.
|
|
630
|
+
* By keeping inline Q4K dequant (f32 arithmetic) the attention path stays
|
|
631
|
+
* numerically correct. FFN projections are unaffected and can safely use
|
|
632
|
+
* the f16-weight GEMV path via remapQ4KDecodeFFNToGemv.
|
|
633
|
+
*
|
|
634
|
+
* Because the derived kernel still references fused_matmul_q4.wgsl,
|
|
635
|
+
* isKernelPathFusedQ4K stays true and the weight loader remains in
|
|
636
|
+
* mixed-materialization mode (Q4K retained for attention, f16 for FFN).
|
|
637
|
+
*
|
|
638
|
+
*/
|
|
639
|
+
export function remapQ4KDecodeAttentionToFusedQ4KGemv(graph, ctx) {
|
|
640
|
+
if (ctx.activationDtype === 'f16') {
|
|
641
|
+
return null;
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
const decodeSteps = graph.decode || [];
|
|
645
|
+
const attnFusedKeys = new Set();
|
|
646
|
+
for (const step of decodeSteps) {
|
|
647
|
+
if (!Array.isArray(step)) continue;
|
|
648
|
+
const op = step[0];
|
|
649
|
+
if (!ATTENTION_PROJECTION_OPS.has(op)) continue;
|
|
650
|
+
const kernelKey = step[1];
|
|
651
|
+
const entry = graph.kernels[kernelKey];
|
|
652
|
+
if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
|
|
653
|
+
attnFusedKeys.add(kernelKey);
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
if (attnFusedKeys.size === 0) {
|
|
657
|
+
return null;
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
const result = cloneGraph(graph);
|
|
661
|
+
const keyMap = new Map();
|
|
662
|
+
|
|
663
|
+
for (const key of attnFusedKeys) {
|
|
664
|
+
const newKey = deriveKernelKey(result.kernels, key, '_gemv');
|
|
665
|
+
result.kernels[newKey] = deriveKernelEntry(
|
|
666
|
+
result.kernels[key],
|
|
667
|
+
'fused_matmul_q4.wgsl',
|
|
668
|
+
'main_gemv',
|
|
669
|
+
null
|
|
670
|
+
);
|
|
671
|
+
keyMap.set(key, newKey);
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
// Only remap attention projection steps, leave FFN steps unchanged.
|
|
675
|
+
result.decode = result.decode.map((step) => {
|
|
676
|
+
if (!Array.isArray(step)) return step;
|
|
677
|
+
if (!ATTENTION_PROJECTION_OPS.has(step[0])) return step;
|
|
678
|
+
const replacement = keyMap.get(step[1]);
|
|
679
|
+
if (replacement !== undefined) {
|
|
680
|
+
const newStep = [...step];
|
|
681
|
+
newStep[1] = replacement;
|
|
682
|
+
return newStep;
|
|
683
|
+
}
|
|
684
|
+
return step;
|
|
685
|
+
});
|
|
686
|
+
|
|
687
|
+
return result;
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
// =============================================================================
|
|
691
|
+
// Transform: remapQ4KDecodeFFNToGemv (diagnostic)
|
|
692
|
+
// =============================================================================
|
|
693
|
+
|
|
694
|
+
const FFN_PROJECTION_OPS = new Set(['gate_proj', 'up_proj', 'down_proj']);
|
|
695
|
+
|
|
696
|
+
/*
|
|
697
|
+
* Replace fused Q4K FFN-ONLY decode projection kernels with GEMV subgroup
|
|
698
|
+
* variants, leaving attention projections (q/k/v/o_proj) as fused Q4K.
|
|
699
|
+
*
|
|
700
|
+
* Diagnostic complement to `remapQ4KDecodeAttentionToGemv`. Together these
|
|
701
|
+
* two transforms isolate whether the GEMV decode regression originates in
|
|
702
|
+
* the attention or FFN projection path. Because attention ops keep their
|
|
703
|
+
* fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight loader
|
|
704
|
+
* remains in mixed-materialization mode.
|
|
705
|
+
*
|
|
706
|
+
*/
|
|
707
|
+
export function remapQ4KDecodeFFNToGemv(graph, ctx) {
|
|
708
|
+
if (ctx.activationDtype === 'f16') {
|
|
709
|
+
return null;
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
const decodeSteps = graph.decode || [];
|
|
713
|
+
const ffnFusedKeys = new Set();
|
|
714
|
+
for (const step of decodeSteps) {
|
|
715
|
+
if (!Array.isArray(step)) continue;
|
|
716
|
+
const op = step[0];
|
|
717
|
+
if (!FFN_PROJECTION_OPS.has(op)) continue;
|
|
718
|
+
const kernelKey = step[1];
|
|
719
|
+
const entry = graph.kernels[kernelKey];
|
|
720
|
+
if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
|
|
721
|
+
ffnFusedKeys.add(kernelKey);
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
if (ffnFusedKeys.size === 0) {
|
|
725
|
+
return null;
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
const result = cloneGraph(graph);
|
|
729
|
+
const keyMap = new Map();
|
|
730
|
+
|
|
731
|
+
for (const key of ffnFusedKeys) {
|
|
732
|
+
const newKey = deriveKernelKey(result.kernels, key, '_ffn_gemv');
|
|
733
|
+
result.kernels[newKey] = deriveKernelEntry(
|
|
734
|
+
result.kernels[key],
|
|
735
|
+
'matmul_gemv_subgroup.wgsl',
|
|
736
|
+
'main_multicol',
|
|
737
|
+
null
|
|
738
|
+
);
|
|
739
|
+
keyMap.set(key, newKey);
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
// Only remap FFN projection steps, leave attention steps unchanged.
|
|
743
|
+
result.decode = result.decode.map((step) => {
|
|
744
|
+
if (!Array.isArray(step)) return step;
|
|
745
|
+
if (!FFN_PROJECTION_OPS.has(step[0])) return step;
|
|
746
|
+
const replacement = keyMap.get(step[1]);
|
|
747
|
+
if (replacement !== undefined) {
|
|
748
|
+
const newStep = [...step];
|
|
749
|
+
newStep[1] = replacement;
|
|
750
|
+
return newStep;
|
|
751
|
+
}
|
|
752
|
+
return step;
|
|
753
|
+
});
|
|
754
|
+
|
|
755
|
+
return result;
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
// =============================================================================
|
|
759
|
+
// Transform: useQwenDecodeF16Matmuls
|
|
760
|
+
// =============================================================================
|
|
761
|
+
|
|
762
|
+
/*
|
|
763
|
+
* Narrow selected Qwen decode matmuls onto explicit f16-input/f16-output
|
|
764
|
+
* kernels while keeping the manifest-wide f32 activation contract intact.
|
|
765
|
+
*
|
|
766
|
+
* This transform is intentionally selective:
|
|
767
|
+
* - FFN gate/up decode matmuls switch to f16a so decode can bypass the slow
|
|
768
|
+
* fused-q4k FFN path when capability policy opts in.
|
|
769
|
+
* - LM head decode switches to the subgroup f16a GEMV path.
|
|
770
|
+
*
|
|
771
|
+
* FFN down remains on the f32-output contract so the layer residual path stays
|
|
772
|
+
* numerically aligned with the manifest-owned activation dtype.
|
|
773
|
+
*
|
|
774
|
+
*/
|
|
775
|
+
|
|
776
|
+
// =============================================================================
|
|
777
|
+
// Transform: useQwenF16PrimaryMatmuls
|
|
778
|
+
// =============================================================================
|
|
779
|
+
|
|
780
|
+
/*
|
|
781
|
+
* Select the Qwen 3.5 small-model execution graph's selective f16 probe lane
|
|
782
|
+
* when the runtime explicitly requests f16 activations.
|
|
783
|
+
*
|
|
784
|
+
* This transform narrows the decode projection and LM-head path while keeping
|
|
785
|
+
* `o_proj` on the stable manifest-owned f32-output kernel. The prefill path is
|
|
786
|
+
* already authored in the manifest on WideTile/head256 kernels and remains
|
|
787
|
+
* manifest-owned. The residual stream feeds directly into the next RMSNorm, and
|
|
788
|
+
* the Qwen 3.5 0.8B promoted f16 lane becomes numerically unstable when
|
|
789
|
+
* `o_proj` writes f16 there (observed first at the first full-attention block's
|
|
790
|
+
* post-attention RMSNorm).
|
|
791
|
+
*
|
|
792
|
+
*/
|
|
793
|
+
|
|
794
|
+
// =============================================================================
|
|
795
|
+
// Transform: useQwen36F16Activations
|
|
796
|
+
// =============================================================================
|
|
797
|
+
|
|
798
|
+
/*
|
|
799
|
+
* Promote the Qwen 3.6 27B Q4K graph onto its additive all-f16 sibling lane.
|
|
800
|
+
*
|
|
801
|
+
*/
|
|
802
|
+
|
|
803
|
+
// =============================================================================
|
|
804
|
+
// Transform: useGemma4Int4PleSelectiveF16Decode
|
|
805
|
+
// =============================================================================
|
|
806
|
+
|
|
807
|
+
/*
|
|
808
|
+
* Promote only Gemma 4 E2B INT4 PLE decode Q/K/V and online attention onto
|
|
809
|
+
* explicit f16 kernels. Prefill remains on manifest-owned f16kv fixed-head
|
|
810
|
+
* attention because the repository does not currently have pure-f16 head256 or
|
|
811
|
+
* head512 prefill kernels.
|
|
812
|
+
*
|
|
813
|
+
*/
|
|
814
|
+
|
|
815
|
+
// =============================================================================
|
|
816
|
+
// Transform: useGemma4TextF16Activations
|
|
817
|
+
// =============================================================================
|
|
818
|
+
|
|
819
|
+
export function useGemma4TextF16Activations(graph, ctx) {
|
|
820
|
+
return useGemma4TextF16ActivationsForLane(graph, ctx, { stableTextBoundary: false });
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
export function useGemma412BTextF16Activations(graph, ctx) {
|
|
824
|
+
return useGemma4TextF16ActivationsForLane(graph, ctx, { stableTextBoundary: true });
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
export function useGemma431BTextF16Activations(graph, ctx) {
|
|
828
|
+
return useGemma4TextF16Activations(graph, ctx);
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
// =============================================================================
|
|
832
|
+
// Transform: useGemma4Int4PleAf16Activations
|
|
833
|
+
// =============================================================================
|
|
834
|
+
|
|
835
|
+
/*
|
|
836
|
+
* Promote the Gemma 4 E2B INT4-PLE Q4K graph onto the all-f16 lane via the
|
|
837
|
+
* weights-ref sibling manifest gemma-4-e2b-it-q4k-ehf16-af16-int4ple. Mirrors
|
|
838
|
+
* useGemma4TextF16Activations: same Q4 weight capsule, kernels narrowed to f16
|
|
839
|
+
* activations, prefill projections promoted from widetile to widetile_f16a,
|
|
840
|
+
* decode projections to multicol_f16a, lm_head/sample/final_norm to their f16
|
|
841
|
+
* counterparts. Apple Metal stays disabled at the capability layer because the
|
|
842
|
+
* fused-q4k+f16 kernel pool produces NaN at L0.ffn_down on metal-3.
|
|
843
|
+
*
|
|
844
|
+
*/
|
|
845
|
+
|
|
846
|
+
// =============================================================================
|
|
847
|
+
// Composition
|
|
848
|
+
// =============================================================================
|
|
849
|
+
|
|
850
|
+
/*
|
|
851
|
+
* Compose multiple transforms into a single transform function.
|
|
852
|
+
*
|
|
853
|
+
* Each transform is applied sequentially. If a transform returns null
|
|
854
|
+
* (not applicable), the graph passes through unchanged.
|
|
855
|
+
*
|
|
856
|
+
*/
|
|
857
|
+
export function composeTransforms(...transforms) {
|
|
858
|
+
return (graph, ctx) => {
|
|
859
|
+
let current = graph;
|
|
860
|
+
for (const transform of transforms) {
|
|
861
|
+
const result = transform(current, ctx);
|
|
862
|
+
if (result !== null && result !== undefined) {
|
|
863
|
+
current = result;
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
return current;
|
|
867
|
+
};
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
// Session-only capability transform marker. The execution-v1 compiler applies
|
|
871
|
+
// the matching runtime session patch; the graph itself is intentionally stable.
|
|
872
|
+
export function disableRetainQ4KMaterialization() {
|
|
873
|
+
return null;
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
/*
|
|
877
|
+
* Fail-closed sentinel transform. A capability rule installs this when the
|
|
878
|
+
* matched (modelId, runtime profile) combination is contradictory — for
|
|
879
|
+
* example, an af32 manifest variant paired with a runtime profile that
|
|
880
|
+
* demands f16 activations. The matcher reaches this transform only when the
|
|
881
|
+
* earlier manifest-binding gate has been bypassed; throwing here keeps the
|
|
882
|
+
* lane-confusion door shut at the capability layer too.
|
|
883
|
+
*
|
|
884
|
+
*/
|
|
885
|
+
export function failClosedLaneMismatch(_graph, ctx) {
|
|
886
|
+
const modelId = ctx?.modelId ?? 'unknown';
|
|
887
|
+
const activationDtype = ctx?.activationDtype ?? 'unknown';
|
|
888
|
+
throw new Error(
|
|
889
|
+
`Capability resolver: lane mismatch for "${modelId}" (activationDtype=${activationDtype}). ` +
|
|
890
|
+
'The manifest variant tag is the lane identity — load the manifest variant ' +
|
|
891
|
+
'whose compute lane matches the runtime profile.'
|
|
892
|
+
);
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
// =============================================================================
|
|
896
|
+
// Registry
|
|
897
|
+
// =============================================================================
|
|
898
|
+
|
|
899
|
+
export const TRANSFORMS = Object.freeze({
|
|
900
|
+
narrowToF16Activations,
|
|
901
|
+
removeSubgroups,
|
|
902
|
+
widenToF32Activations,
|
|
903
|
+
widenToF32CorrectnessFallback,
|
|
904
|
+
widenAttentionKvToF32,
|
|
905
|
+
swapPrefillAttention,
|
|
906
|
+
useHead256SmallPrefillAttention,
|
|
907
|
+
useHead256PrefillAttention,
|
|
908
|
+
widenProjectionWeightsToF32,
|
|
909
|
+
remapDenseQ4KPrefillToQ4Native,
|
|
910
|
+
remapQ4KPrefillToDense,
|
|
911
|
+
useLinearDecodeProjectionF16,
|
|
912
|
+
remapQ4KDecodeToGemv,
|
|
913
|
+
remapQ4KDecodeAttentionToGemv,
|
|
914
|
+
remapQ4KDecodeAttentionToFusedQ4KGemv,
|
|
915
|
+
remapQ4KDecodeFFNToGemv,
|
|
916
|
+
disableRetainQ4KMaterialization,
|
|
917
|
+
useQwenF16PrimaryMatmuls,
|
|
918
|
+
useQwen36F16Activations,
|
|
919
|
+
useQwenDecodeF16Matmuls,
|
|
920
|
+
useGemma4Int4PleSelectiveF16Decode,
|
|
921
|
+
useGemma4TextF16Activations,
|
|
922
|
+
useGemma412BTextF16Activations,
|
|
923
|
+
useGemma431BTextF16Activations,
|
|
924
|
+
useGemma4Int4PleAf16Activations,
|
|
925
|
+
failClosedLaneMismatch,
|
|
926
|
+
composeTransforms,
|
|
927
|
+
});
|