doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
{
|
|
2
2
|
"capabilityTransforms": [
|
|
3
|
+
{
|
|
4
|
+
"match": {
|
|
5
|
+
"modelId": "gemma-4-e2b-it-q4k-ehf16-af16-int4ple",
|
|
6
|
+
"platformVendor": "apple"
|
|
7
|
+
},
|
|
8
|
+
"transforms": ["failClosedLaneMismatch"],
|
|
9
|
+
"reason": "af16-int4ple lane requires retainQ4KMaterialization=true to dispatch the fused-q4k+f16 widetile pool, which produces NaN at L0.ffn_down on Apple Metal — load the af32 manifest variant instead"
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
"match": {
|
|
13
|
+
"modelId": { "startsWith": "gemma-4-e2b-it-q4k-" },
|
|
14
|
+
"platformVendor": "apple",
|
|
15
|
+
"retainQ4KMaterialization": true
|
|
16
|
+
},
|
|
17
|
+
"transforms": ["disableRetainQ4KMaterialization"],
|
|
18
|
+
"reason": "Fused-q4k FFN kernels under retain=mixed corrupt subsequent-layer attention compute on Apple Metal (NaN at L1.attn_core_out and beyond). Standalone-q4k projection kernels are clean; only the fused gate+up+down dispatch is the perturbation source."
|
|
19
|
+
},
|
|
3
20
|
{
|
|
4
21
|
"match": {
|
|
5
22
|
"modelId": "qwen-3-5-0-8b-q4k-ehaf16",
|
|
@@ -10,6 +27,76 @@
|
|
|
10
27
|
"transforms": ["useQwenF16PrimaryMatmuls"],
|
|
11
28
|
"reason": "Runtime requested the promoted f16 primary lane for Qwen 3.5 0.8B"
|
|
12
29
|
},
|
|
30
|
+
{
|
|
31
|
+
"match": {
|
|
32
|
+
"modelId": "qwen-3-6-27b-q4k-eaf16",
|
|
33
|
+
"activationDtype": "f16",
|
|
34
|
+
"mathDtype": "f16",
|
|
35
|
+
"accumDtype": "f16",
|
|
36
|
+
"hasSubgroups": true,
|
|
37
|
+
"hasF16": true
|
|
38
|
+
},
|
|
39
|
+
"transforms": ["useQwen36F16Activations"],
|
|
40
|
+
"reason": "Runtime requested the experimental all-f16 lane for Qwen 3.6 27B"
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"match": {
|
|
44
|
+
"modelId": "gemma-4-e2b-it-q4k-ehf16-af32-int4ple",
|
|
45
|
+
"activationDtype": "f16",
|
|
46
|
+
"hasSubgroups": true,
|
|
47
|
+
"hasF16": true
|
|
48
|
+
},
|
|
49
|
+
"transforms": ["useGemma4Int4PleSelectiveF16Decode"],
|
|
50
|
+
"reason": "Runtime requested the experimental selective f16 decode lane for Gemma 4 E2B INT4 PLE"
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"match": {
|
|
54
|
+
"modelId": "gemma-4-31b-it-text-q4k-ehf16-af16",
|
|
55
|
+
"activationDtype": "f16",
|
|
56
|
+
"mathDtype": "f16",
|
|
57
|
+
"accumDtype": "f16",
|
|
58
|
+
"hasSubgroups": true,
|
|
59
|
+
"hasF16": true
|
|
60
|
+
},
|
|
61
|
+
"transforms": ["useGemma431BTextF16Activations"],
|
|
62
|
+
"reason": "af16 manifest variant on shader-f16 GPU — dispatch the all-f16 lane"
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"match": {
|
|
66
|
+
"modelId": "gemma-4-e2b-it-q4k-ehf16-af16-int4ple",
|
|
67
|
+
"activationDtype": "f16",
|
|
68
|
+
"mathDtype": "f16",
|
|
69
|
+
"accumDtype": "f16",
|
|
70
|
+
"hasSubgroups": true,
|
|
71
|
+
"hasF16": true
|
|
72
|
+
},
|
|
73
|
+
"transforms": ["useGemma4Int4PleAf16Activations"],
|
|
74
|
+
"reason": "af16-int4ple manifest variant on shader-f16 GPU — dispatch the all-f16 lane (q4_widetile_f16a + decode multicol_f16a + lm_head_gemv_subgroup_f16a)"
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
"match": {
|
|
78
|
+
"modelId": "gemma-4-31b-it-text-q4k-ehf16-af32",
|
|
79
|
+
"activationDtype": "f16",
|
|
80
|
+
"mathDtype": "f16",
|
|
81
|
+
"accumDtype": "f16",
|
|
82
|
+
"hasSubgroups": true,
|
|
83
|
+
"hasF16": true
|
|
84
|
+
},
|
|
85
|
+
"transforms": ["failClosedLaneMismatch"],
|
|
86
|
+
"reason": "af32 manifest cannot dispatch f16 activations — load the af16 manifest variant or pick an f32 runtime profile"
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
"match": {
|
|
90
|
+
"modelId": { "contains": "gemma-4-e2b" },
|
|
91
|
+
"activationDtype": "f32",
|
|
92
|
+
"headDim": 256,
|
|
93
|
+
"kvDtype": "f16",
|
|
94
|
+
"hasF16": true,
|
|
95
|
+
"maxWorkgroupStorageSize": { "gte": 16384 }
|
|
96
|
+
},
|
|
97
|
+
"transforms": ["useHead256SmallPrefillAttention"],
|
|
98
|
+
"reason": "Gemma 4 E2B promotes only small-tile prefill attention onto the head256 kernel on capable GPUs"
|
|
99
|
+
},
|
|
13
100
|
{
|
|
14
101
|
"match": {
|
|
15
102
|
"activationDtype": "f16",
|
|
@@ -85,10 +85,20 @@
|
|
|
85
85
|
"match": {
|
|
86
86
|
"hasGpuSplitPerLayerInputs": true
|
|
87
87
|
},
|
|
88
|
-
"value":
|
|
88
|
+
"value": 8
|
|
89
89
|
},
|
|
90
90
|
{ "match": {}, "value": null }
|
|
91
91
|
],
|
|
92
|
+
"prefillRecorderChunkLayers": [
|
|
93
|
+
{
|
|
94
|
+
"match": {
|
|
95
|
+
"hasGpuSplitPerLayerInputs": true,
|
|
96
|
+
"numTokens": { "lte": 32 }
|
|
97
|
+
},
|
|
98
|
+
"value": 8
|
|
99
|
+
},
|
|
100
|
+
{ "match": {}, "value": 4 }
|
|
101
|
+
],
|
|
92
102
|
"finitenessFallbackActivationDtype": [
|
|
93
103
|
{ "match": {}, "value": "f32" }
|
|
94
104
|
],
|
|
@@ -40,7 +40,9 @@
|
|
|
40
40
|
"dtypeSupported": true,
|
|
41
41
|
"weightDtype": "q4k",
|
|
42
42
|
"activationDtype": "f32",
|
|
43
|
-
"hiddenSizeAligned32": true
|
|
43
|
+
"hiddenSizeAligned32": true,
|
|
44
|
+
"batchSize": { "lte": 4 },
|
|
45
|
+
"useDoubleWideMlp": false
|
|
44
46
|
},
|
|
45
47
|
"value": true
|
|
46
48
|
},
|
|
@@ -56,7 +58,9 @@
|
|
|
56
58
|
"dtypeSupported": true,
|
|
57
59
|
"hasQ4KMaterialization": true,
|
|
58
60
|
"activationDtype": "f32",
|
|
59
|
-
"hiddenSizeAligned32": true
|
|
61
|
+
"hiddenSizeAligned32": true,
|
|
62
|
+
"batchSize": { "lte": 4 },
|
|
63
|
+
"useDoubleWideMlp": false
|
|
60
64
|
},
|
|
61
65
|
"value": true
|
|
62
66
|
},
|
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"gpuLoaderPath": [
|
|
3
|
+
{
|
|
4
|
+
"match": {
|
|
5
|
+
"dtype": { "in": ["Q4_K_M", "Q4_K"] },
|
|
6
|
+
"q4kDenseExceedsBindingLimit": true,
|
|
7
|
+
"q4kLimitFallbackEligible": true
|
|
8
|
+
},
|
|
9
|
+
"value": "q4k_fused"
|
|
10
|
+
},
|
|
3
11
|
{ "match": { "dtype": { "in": ["Q4_K_M", "Q4_K"] }, "q4kMaterializationMode": "mixed" }, "value": "q4k_mixed" },
|
|
12
|
+
{ "match": { "dtype": { "in": ["Q4_K_M", "Q4_K"] }, "requiresFusedQ4KRole": true }, "value": "q4k_fused" },
|
|
4
13
|
{ "match": { "dtype": { "in": ["Q4_K_M", "Q4_K"] }, "useFusedQ4K": true }, "value": "q4k_fused" },
|
|
5
14
|
{
|
|
6
15
|
"match": {
|
|
@@ -1,14 +1,59 @@
|
|
|
1
1
|
import { selectByRules } from '../gpu/kernels/rule-matcher.js';
|
|
2
2
|
import { buildInferenceExecutionRulesContractArtifact } from './execution-rules-contract-check.js';
|
|
3
3
|
import { buildLayerPatternContractArtifact } from './layer-pattern-contract-check.js';
|
|
4
|
-
import {
|
|
4
|
+
import { cloneJsonValue as cloneRuleValue } from '../utils/clone-json.js';
|
|
5
|
+
|
|
6
|
+
// Rule files imported statically (JSON import attributes). Sync at module
|
|
7
|
+
// load time, no top-level await, and bundle-friendly. Requires Node >=22
|
|
8
|
+
// and any bundler with JSON-module support (Vite/Rollup/esbuild/webpack5).
|
|
9
|
+
import attentionRules from './kernels/attention.rules.json' with { type: 'json' };
|
|
10
|
+
import conv2dRules from './kernels/conv2d.rules.json' with { type: 'json' };
|
|
11
|
+
import depthwiseConv2dRules from './kernels/depthwise-conv2d.rules.json' with { type: 'json' };
|
|
12
|
+
import dequantRules from './kernels/dequant.rules.json' with { type: 'json' };
|
|
13
|
+
import energyRules from './kernels/energy.rules.json' with { type: 'json' };
|
|
14
|
+
import fusedFfnRules from './kernels/fused-ffn.rules.json' with { type: 'json' };
|
|
15
|
+
import fusedMatmulResidualRules from './kernels/fused-matmul-residual.rules.json' with { type: 'json' };
|
|
16
|
+
import fusedMatmulRmsnormRules from './kernels/fused-matmul-rmsnorm.rules.json' with { type: 'json' };
|
|
17
|
+
import gatherRules from './kernels/gather.rules.json' with { type: 'json' };
|
|
18
|
+
import geluRules from './kernels/gelu.rules.json' with { type: 'json' };
|
|
19
|
+
import groupedPointwiseConv2dRules from './kernels/grouped-pointwise-conv2d.rules.json' with { type: 'json' };
|
|
20
|
+
import groupnormRules from './kernels/groupnorm.rules.json' with { type: 'json' };
|
|
21
|
+
import kvQuantizeRules from './kernels/kv_quantize.rules.json' with { type: 'json' };
|
|
22
|
+
import layernormRules from './kernels/layernorm.rules.json' with { type: 'json' };
|
|
23
|
+
import matmulRules from './kernels/matmul.rules.json' with { type: 'json' };
|
|
24
|
+
import kernelMoeRules from './kernels/moe.rules.json' with { type: 'json' };
|
|
25
|
+
import kernelMoeGptOssRules from './kernels/moe.rules.gptoss.json' with { type: 'json' };
|
|
26
|
+
import kernelMoeMixtralRules from './kernels/moe.rules.mixtral.json' with { type: 'json' };
|
|
27
|
+
import modulateRules from './kernels/modulate.rules.json' with { type: 'json' };
|
|
28
|
+
import pixelShuffleRules from './kernels/pixel_shuffle.rules.json' with { type: 'json' };
|
|
29
|
+
import repeatChannelsRules from './kernels/repeat-channels.rules.json' with { type: 'json' };
|
|
30
|
+
import reluRules from './kernels/relu.rules.json' with { type: 'json' };
|
|
31
|
+
import residualRules from './kernels/residual.rules.json' with { type: 'json' };
|
|
32
|
+
import rmsnormRules from './kernels/rmsnorm.rules.json' with { type: 'json' };
|
|
33
|
+
import ropeRules from './kernels/rope.rules.json' with { type: 'json' };
|
|
34
|
+
import linearAttentionRules from './kernels/linear-attention.rules.json' with { type: 'json' };
|
|
35
|
+
import sampleRules from './kernels/sample.rules.json' with { type: 'json' };
|
|
36
|
+
import scaleRules from './kernels/scale.rules.json' with { type: 'json' };
|
|
37
|
+
import siluRules from './kernels/silu.rules.json' with { type: 'json' };
|
|
38
|
+
import splitQkvRules from './kernels/split-qkv.rules.json' with { type: 'json' };
|
|
39
|
+
import splitQgRules from './kernels/split-qg.rules.json' with { type: 'json' };
|
|
40
|
+
import softmaxRules from './kernels/softmax.rules.json' with { type: 'json' };
|
|
41
|
+
import upsample2dRules from './kernels/upsample2d.rules.json' with { type: 'json' };
|
|
42
|
+
import configRules from './inference/config.rules.json' with { type: 'json' };
|
|
43
|
+
import inferenceExecutionRules from './inference/execution.rules.json' with { type: 'json' };
|
|
44
|
+
import inferenceAttentionRules from './inference/attention.rules.json' with { type: 'json' };
|
|
45
|
+
import dtypeRules from './inference/dtype.rules.json' with { type: 'json' };
|
|
46
|
+
import ffnRules from './inference/ffn.rules.json' with { type: 'json' };
|
|
47
|
+
import layerRules from './inference/layer.rules.json' with { type: 'json' };
|
|
48
|
+
import layerPatternRules from './inference/layer-pattern.rules.json' with { type: 'json' };
|
|
49
|
+
import inferenceMoeRules from './inference/moe.rules.json' with { type: 'json' };
|
|
50
|
+
import tokenizerRules from './converter/tokenizer.rules.json' with { type: 'json' };
|
|
51
|
+
import tensorRolesRules from './converter/tensor-roles.rules.json' with { type: 'json' };
|
|
52
|
+
import converterExecutionRules from './converter/execution.rules.json' with { type: 'json' };
|
|
53
|
+
import loaderWeightRules from './loader/weights.rules.json' with { type: 'json' };
|
|
54
|
+
import tensorLoaderRules from './loader/tensor-loader.rules.json' with { type: 'json' };
|
|
55
|
+
import toolingCommandRuntimeRules from './tooling/command-runtime.rules.json' with { type: 'json' };
|
|
5
56
|
|
|
6
|
-
function cloneRuleValue(value) {
|
|
7
|
-
if (typeof structuredClone === 'function') {
|
|
8
|
-
return structuredClone(value);
|
|
9
|
-
}
|
|
10
|
-
return JSON.parse(JSON.stringify(value));
|
|
11
|
-
}
|
|
12
57
|
|
|
13
58
|
// deepFreeze assumes all values in the tree are plain objects, arrays, or
|
|
14
59
|
// primitives. Typed arrays, Maps, Sets, and other exotic objects will be
|
|
@@ -24,73 +69,6 @@ function deepFreeze(value, seen = new WeakSet()) {
|
|
|
24
69
|
}
|
|
25
70
|
return Object.freeze(value);
|
|
26
71
|
}
|
|
27
|
-
|
|
28
|
-
const ruleLoadFailures = [];
|
|
29
|
-
|
|
30
|
-
async function safeLoadJson(path, label) {
|
|
31
|
-
try {
|
|
32
|
-
return await loadJson(path, import.meta.url, 'Failed to load rules');
|
|
33
|
-
} catch (error) {
|
|
34
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
35
|
-
console.warn(`RuleRegistry: failed to load "${path}": ${message}`);
|
|
36
|
-
ruleLoadFailures.push(path);
|
|
37
|
-
return null;
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
const attentionRules = await safeLoadJson('./kernels/attention.rules.json');
|
|
42
|
-
const conv2dRules = await safeLoadJson('./kernels/conv2d.rules.json');
|
|
43
|
-
const depthwiseConv2dRules = await safeLoadJson('./kernels/depthwise-conv2d.rules.json');
|
|
44
|
-
const dequantRules = await safeLoadJson('./kernels/dequant.rules.json');
|
|
45
|
-
const energyRules = await safeLoadJson('./kernels/energy.rules.json');
|
|
46
|
-
const fusedFfnRules = await safeLoadJson('./kernels/fused-ffn.rules.json');
|
|
47
|
-
const fusedMatmulResidualRules = await safeLoadJson('./kernels/fused-matmul-residual.rules.json');
|
|
48
|
-
const fusedMatmulRmsnormRules = await safeLoadJson('./kernels/fused-matmul-rmsnorm.rules.json');
|
|
49
|
-
const gatherRules = await safeLoadJson('./kernels/gather.rules.json');
|
|
50
|
-
const geluRules = await safeLoadJson('./kernels/gelu.rules.json');
|
|
51
|
-
const groupedPointwiseConv2dRules = await safeLoadJson('./kernels/grouped-pointwise-conv2d.rules.json');
|
|
52
|
-
const groupnormRules = await safeLoadJson('./kernels/groupnorm.rules.json');
|
|
53
|
-
const kvQuantizeRules = await safeLoadJson('./kernels/kv_quantize.rules.json');
|
|
54
|
-
const layernormRules = await safeLoadJson('./kernels/layernorm.rules.json');
|
|
55
|
-
const matmulRules = await safeLoadJson('./kernels/matmul.rules.json');
|
|
56
|
-
const kernelMoeRules = await safeLoadJson('./kernels/moe.rules.json');
|
|
57
|
-
const kernelMoeGptOssRules = await safeLoadJson('./kernels/moe.rules.gptoss.json');
|
|
58
|
-
const kernelMoeMixtralRules = await safeLoadJson('./kernels/moe.rules.mixtral.json');
|
|
59
|
-
const modulateRules = await safeLoadJson('./kernels/modulate.rules.json');
|
|
60
|
-
const pixelShuffleRules = await safeLoadJson('./kernels/pixel_shuffle.rules.json');
|
|
61
|
-
const repeatChannelsRules = await safeLoadJson('./kernels/repeat-channels.rules.json');
|
|
62
|
-
const reluRules = await safeLoadJson('./kernels/relu.rules.json');
|
|
63
|
-
const residualRules = await safeLoadJson('./kernels/residual.rules.json');
|
|
64
|
-
const rmsnormRules = await safeLoadJson('./kernels/rmsnorm.rules.json');
|
|
65
|
-
const ropeRules = await safeLoadJson('./kernels/rope.rules.json');
|
|
66
|
-
const sanaLinearAttentionRules = await safeLoadJson('./kernels/sana-linear-attention.rules.json');
|
|
67
|
-
const sampleRules = await safeLoadJson('./kernels/sample.rules.json');
|
|
68
|
-
const scaleRules = await safeLoadJson('./kernels/scale.rules.json');
|
|
69
|
-
const siluRules = await safeLoadJson('./kernels/silu.rules.json');
|
|
70
|
-
const splitQkvRules = await safeLoadJson('./kernels/split-qkv.rules.json');
|
|
71
|
-
const splitQgRules = await safeLoadJson('./kernels/split-qg.rules.json');
|
|
72
|
-
const softmaxRules = await safeLoadJson('./kernels/softmax.rules.json');
|
|
73
|
-
const upsample2dRules = await safeLoadJson('./kernels/upsample2d.rules.json');
|
|
74
|
-
const configRules = await safeLoadJson('./inference/config.rules.json');
|
|
75
|
-
const inferenceExecutionRules = await safeLoadJson('./inference/execution.rules.json');
|
|
76
|
-
const inferenceAttentionRules = await safeLoadJson('./inference/attention.rules.json');
|
|
77
|
-
const dtypeRules = await safeLoadJson('./inference/dtype.rules.json');
|
|
78
|
-
const ffnRules = await safeLoadJson('./inference/ffn.rules.json');
|
|
79
|
-
const layerRules = await safeLoadJson('./inference/layer.rules.json');
|
|
80
|
-
const layerPatternRules = await safeLoadJson('./inference/layer-pattern.rules.json');
|
|
81
|
-
const inferenceMoeRules = await safeLoadJson('./inference/moe.rules.json');
|
|
82
|
-
const tokenizerRules = await safeLoadJson('./converter/tokenizer.rules.json');
|
|
83
|
-
const tensorRolesRules = await safeLoadJson('./converter/tensor-roles.rules.json');
|
|
84
|
-
const converterExecutionRules = await safeLoadJson('./converter/execution.rules.json');
|
|
85
|
-
const loaderWeightRules = await safeLoadJson('./loader/weights.rules.json');
|
|
86
|
-
const tensorLoaderRules = await safeLoadJson('./loader/tensor-loader.rules.json');
|
|
87
|
-
const toolingCommandRuntimeRules = await safeLoadJson('./tooling/command-runtime.rules.json');
|
|
88
|
-
|
|
89
|
-
if (ruleLoadFailures.length > 0) {
|
|
90
|
-
console.warn(
|
|
91
|
-
`RuleRegistry: ${ruleLoadFailures.length} rule file(s) failed to load: ${ruleLoadFailures.join(', ')}`
|
|
92
|
-
);
|
|
93
|
-
}
|
|
94
72
|
const INFERENCE_EXECUTION_RULES_CONTRACT_ARTIFACT = buildInferenceExecutionRulesContractArtifact(
|
|
95
73
|
inferenceExecutionRules
|
|
96
74
|
);
|
|
@@ -140,7 +118,7 @@ const RULE_SETS = {
|
|
|
140
118
|
residual: residualRules,
|
|
141
119
|
rmsnorm: rmsnormRules,
|
|
142
120
|
rope: ropeRules,
|
|
143
|
-
|
|
121
|
+
linearAttention: linearAttentionRules,
|
|
144
122
|
sample: sampleRules,
|
|
145
123
|
scale: scaleRules,
|
|
146
124
|
silu: siluRules,
|
|
@@ -23,14 +23,23 @@ export interface CreateArtifactStorageContextOptions {
|
|
|
23
23
|
) => AsyncIterable<ArrayBuffer | Uint8Array>;
|
|
24
24
|
readText?: (path: string) => Promise<string | Record<string, unknown> | null | undefined>;
|
|
25
25
|
readBinary?: (path: string) => Promise<ArrayBuffer | Uint8Array | null | undefined>;
|
|
26
|
+
close?: (() => Promise<void>) | null;
|
|
26
27
|
tokenizerJsonPath?: string | null;
|
|
27
28
|
tokenizerModelPath?: string | null;
|
|
29
|
+
tensorsJsonPath?: string | null;
|
|
28
30
|
verifyHashes?: boolean;
|
|
29
31
|
hashesTrusted?: boolean;
|
|
30
32
|
}
|
|
31
33
|
|
|
34
|
+
export interface ArtifactStorageContext extends SourceStorageContext {
|
|
35
|
+
preflight?: () => Promise<void>;
|
|
36
|
+
}
|
|
37
|
+
|
|
32
38
|
export interface CreateHttpArtifactStorageContextOptions {
|
|
33
39
|
verifyHashes?: boolean;
|
|
40
|
+
rangeCacheBlockBytes?: number;
|
|
41
|
+
rangeCacheMaxBytes?: number;
|
|
42
|
+
rangeCacheMinBytes?: number;
|
|
34
43
|
}
|
|
35
44
|
|
|
36
45
|
export declare function getArtifactFormat(
|
|
@@ -39,15 +48,15 @@ export declare function getArtifactFormat(
|
|
|
39
48
|
|
|
40
49
|
export declare function createArtifactStorageContext(
|
|
41
50
|
options: CreateArtifactStorageContextOptions
|
|
42
|
-
):
|
|
51
|
+
): ArtifactStorageContext;
|
|
43
52
|
|
|
44
53
|
export declare function createNodeFileArtifactStorageContext(
|
|
45
54
|
baseUrl: string | null | undefined,
|
|
46
55
|
manifest: RDRRManifest
|
|
47
|
-
):
|
|
56
|
+
): ArtifactStorageContext | null;
|
|
48
57
|
|
|
49
58
|
export declare function createHttpArtifactStorageContext(
|
|
50
59
|
baseUrl: string | null | undefined,
|
|
51
60
|
manifest: RDRRManifest,
|
|
52
61
|
options?: CreateHttpArtifactStorageContextOptions
|
|
53
|
-
):
|
|
62
|
+
): ArtifactStorageContext | null;
|