doppler-gpu 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +19 -15
- package/package.json +77 -29
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +64 -6
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +524 -350
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +12 -2
- package/src/config/schema/doppler.schema.js +108 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +68 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +76 -0
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +107 -0
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +3 -21
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +6 -5
- package/src/gpu/kernels/index.js +3 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +12 -0
- package/src/inference/browser-harness-text-helpers.js +223 -14
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +264 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +22 -27
- package/src/inference/pipelines/text/embed.js +0 -39
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +4 -5
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +8 -3
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +223 -547
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +5 -2
- package/src/inference/pipelines/text/init.js +183 -724
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.js +2 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +120 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +223 -27
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +13 -3
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +11 -3
- package/src/storage/artifact-storage-context.js +231 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +21 -24
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +2 -0
- package/src/tooling/source-runtime-bundle.js +3 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
PER_LAYER_INPUT_PREFETCH_MODES,
|
|
8
8
|
PER_LAYER_INPUT_GPU_UPLOAD_MODES,
|
|
9
9
|
PER_LAYER_INPUT_HOT_CACHE_MODES,
|
|
10
|
+
PREFILL_CHUNK_SUBMIT_MODES,
|
|
10
11
|
} from '../../../config/schema/execution-v1.schema.js';
|
|
11
12
|
|
|
12
13
|
const UNSUPPORTED_RUNTIME_MODEL_TYPES = new Set(['mamba', 'rwkv']);
|
|
@@ -34,7 +35,7 @@ const KNOWN_CHAT_TEMPLATE_TYPES = new Set([
|
|
|
34
35
|
* @param {string} modelId - Model identifier for diagnostic messages.
|
|
35
36
|
* @returns {boolean} True if the type is valid or null.
|
|
36
37
|
*/
|
|
37
|
-
|
|
38
|
+
function validateChatTemplateType(type, modelId) {
|
|
38
39
|
if (type === null || type === undefined) return true;
|
|
39
40
|
if (KNOWN_CHAT_TEMPLATE_TYPES.has(type)) return true;
|
|
40
41
|
throw new Error(
|
|
@@ -196,6 +197,58 @@ function resolvePerLayerInputsSession(inferenceConfig, modelId) {
|
|
|
196
197
|
};
|
|
197
198
|
}
|
|
198
199
|
|
|
200
|
+
function resolveSessionSettings(inferenceConfig, modelId) {
|
|
201
|
+
// All four fields are optional on the manifest today for backwards compatibility;
|
|
202
|
+
// when absent we fall through to the runtime config (getRuntimeConfig()). When
|
|
203
|
+
// present on the manifest, manifest wins unless an explicit runtime profile overrides.
|
|
204
|
+
// The merge layer in src/config/merge.js already applies that precedence — this
|
|
205
|
+
// resolver just validates the manifest-supplied values and normalizes them.
|
|
206
|
+
const session = inferenceConfig?.session;
|
|
207
|
+
const submit = session?.prefillChunkSubmitMode;
|
|
208
|
+
if (submit !== undefined && submit !== null && !PREFILL_CHUNK_SUBMIT_MODES.includes(submit)) {
|
|
209
|
+
throw new Error(
|
|
210
|
+
`Manifest "${modelId}" has invalid inference.session.prefillChunkSubmitMode ` +
|
|
211
|
+
`"${String(submit)}"; expected one of ${PREFILL_CHUNK_SUBMIT_MODES.join(', ')}.`
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
const flash = session?.useFlashPrefillAttention;
|
|
215
|
+
if (flash !== undefined && flash !== null && typeof flash !== 'boolean') {
|
|
216
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFlashPrefillAttention "${String(flash)}"; expected boolean.`);
|
|
217
|
+
}
|
|
218
|
+
const wide = session?.useWideTileQ4KPrefill;
|
|
219
|
+
if (wide !== undefined && wide !== null && typeof wide !== 'boolean') {
|
|
220
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileQ4KPrefill "${String(wide)}"; expected boolean.`);
|
|
221
|
+
}
|
|
222
|
+
const retain = session?.retainQ4KMaterialization;
|
|
223
|
+
if (retain !== undefined && retain !== null && typeof retain !== 'boolean') {
|
|
224
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.retainQ4KMaterialization "${String(retain)}"; expected boolean.`);
|
|
225
|
+
}
|
|
226
|
+
return {
|
|
227
|
+
prefillChunkSubmitMode: submit ?? null,
|
|
228
|
+
useFlashPrefillAttention: flash ?? null,
|
|
229
|
+
useWideTileQ4KPrefill: wide ?? null,
|
|
230
|
+
retainQ4KMaterialization: retain ?? null,
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function resolveLargeWeightsConfig(inferenceConfig, modelId) {
|
|
235
|
+
const lw = inferenceConfig?.largeWeights;
|
|
236
|
+
if (lw === undefined || lw === null) return { gpuResidentOverrides: null };
|
|
237
|
+
if (typeof lw !== 'object' || Array.isArray(lw)) {
|
|
238
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.largeWeights (must be object).`);
|
|
239
|
+
}
|
|
240
|
+
const overrides = lw.gpuResidentOverrides;
|
|
241
|
+
if (overrides !== null && overrides !== undefined) {
|
|
242
|
+
if (!Array.isArray(overrides) || !overrides.every((v) => typeof v === 'string' && v.length > 0)) {
|
|
243
|
+
throw new Error(
|
|
244
|
+
`Manifest "${modelId}" has invalid inference.largeWeights.gpuResidentOverrides; ` +
|
|
245
|
+
`expected array of non-empty strings or null.`
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
return { gpuResidentOverrides: overrides ?? null };
|
|
250
|
+
}
|
|
251
|
+
|
|
199
252
|
// =============================================================================
|
|
200
253
|
// Model Detection Functions
|
|
201
254
|
// =============================================================================
|
|
@@ -541,7 +594,7 @@ export function resolveLayerIntermediateSize(config, layerIdx) {
|
|
|
541
594
|
// =============================================================================
|
|
542
595
|
|
|
543
596
|
|
|
544
|
-
|
|
597
|
+
function hasManifestInference(manifest) {
|
|
545
598
|
return 'inference' in manifest && manifest.inference != null;
|
|
546
599
|
}
|
|
547
600
|
|
|
@@ -1120,7 +1173,7 @@ function parseLinearNormMode(value, sharedFlag = null, modelId = 'unknown') {
|
|
|
1120
1173
|
}
|
|
1121
1174
|
|
|
1122
1175
|
|
|
1123
|
-
|
|
1176
|
+
function toParsedConfigFromMerged(merged, manifest) {
|
|
1124
1177
|
const mergedConfig = merged?.config ?? manifest.config ?? {};
|
|
1125
1178
|
const rawConfig = mergedConfig.text_config ?? mergedConfig;
|
|
1126
1179
|
const config = {
|
|
@@ -1397,6 +1450,49 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1397
1450
|
return Math.trunc(num);
|
|
1398
1451
|
};
|
|
1399
1452
|
|
|
1453
|
+
const resolveTensorOutputRows = (tensorInfo, hiddenSize) => {
|
|
1454
|
+
const shape = Array.isArray(tensorInfo?.shape) ? tensorInfo.shape : null;
|
|
1455
|
+
if (!shape || shape.length < 2) return null;
|
|
1456
|
+
const dim0 = Number(shape[0]);
|
|
1457
|
+
const dim1 = Number(shape[1]);
|
|
1458
|
+
if (!Number.isFinite(dim0) || !Number.isFinite(dim1)) return null;
|
|
1459
|
+
if (dim1 === hiddenSize) return Math.trunc(dim0);
|
|
1460
|
+
if (dim0 === hiddenSize) return Math.trunc(dim1);
|
|
1461
|
+
return null;
|
|
1462
|
+
};
|
|
1463
|
+
|
|
1464
|
+
const deriveGlobalKVHeadsFromManifest = () => {
|
|
1465
|
+
if (!Array.isArray(layerTypes) || archGlobalHeadDim == null) {
|
|
1466
|
+
return null;
|
|
1467
|
+
}
|
|
1468
|
+
const tensors = manifest?.tensors && typeof manifest.tensors === 'object' ? manifest.tensors : null;
|
|
1469
|
+
if (!tensors) return null;
|
|
1470
|
+
const globalLayerIdx = layerTypes.findIndex((layerType) => {
|
|
1471
|
+
const normalized = typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
|
|
1472
|
+
return normalized && normalized !== 'sliding_attention' && normalized !== 'local_attention'
|
|
1473
|
+
&& normalized !== 'local' && normalized !== 'sliding';
|
|
1474
|
+
});
|
|
1475
|
+
if (globalLayerIdx < 0) return null;
|
|
1476
|
+
const layerMarker = `.layers.${globalLayerIdx}.`;
|
|
1477
|
+
for (const [tensorName, tensorInfo] of Object.entries(tensors)) {
|
|
1478
|
+
if (!tensorName.includes(layerMarker) || !tensorName.includes('.self_attn.k_proj.weight')) {
|
|
1479
|
+
continue;
|
|
1480
|
+
}
|
|
1481
|
+
const rows = resolveTensorOutputRows(tensorInfo, arch.hiddenSize);
|
|
1482
|
+
if (rows != null && rows % archGlobalHeadDim === 0) {
|
|
1483
|
+
return rows / archGlobalHeadDim;
|
|
1484
|
+
}
|
|
1485
|
+
}
|
|
1486
|
+
return null;
|
|
1487
|
+
};
|
|
1488
|
+
|
|
1489
|
+
const archNumGlobalKVHeads = parsePositiveInt(
|
|
1490
|
+
arch.numGlobalKeyValueHeads
|
|
1491
|
+
?? arch.numGlobalKVHeads
|
|
1492
|
+
?? config.num_global_key_value_heads
|
|
1493
|
+
?? config.num_global_kv_heads
|
|
1494
|
+
) ?? deriveGlobalKVHeadsFromManifest();
|
|
1495
|
+
|
|
1400
1496
|
const linearNumKeyHeads = parsePositiveInt(arch.linearNumKeyHeads ?? config.linear_num_key_heads);
|
|
1401
1497
|
const linearNumValueHeads = parsePositiveInt(arch.linearNumValueHeads ?? config.linear_num_value_heads);
|
|
1402
1498
|
const linearKeyHeadDim = parsePositiveInt(arch.linearKeyHeadDim ?? config.linear_key_head_dim);
|
|
@@ -1414,6 +1510,8 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1414
1510
|
? 'replay_prefill'
|
|
1415
1511
|
: 'incremental';
|
|
1416
1512
|
const perLayerInputsSession = resolvePerLayerInputsSession(inf, merged.modelId);
|
|
1513
|
+
const sessionSettings = resolveSessionSettings(inf, merged.modelId);
|
|
1514
|
+
const largeWeightsConfig = resolveLargeWeightsConfig(inf, merged.modelId);
|
|
1417
1515
|
|
|
1418
1516
|
return {
|
|
1419
1517
|
numLayers: arch.numLayers,
|
|
@@ -1423,6 +1521,7 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1423
1521
|
maxIntermediateSize,
|
|
1424
1522
|
numHeads: archNumHeads,
|
|
1425
1523
|
numKVHeads: archNumKVHeads,
|
|
1524
|
+
numGlobalKVHeads: archNumGlobalKVHeads,
|
|
1426
1525
|
headDim: archHeadDim,
|
|
1427
1526
|
globalHeadDim: archGlobalHeadDim,
|
|
1428
1527
|
vocabSize: arch.vocabSize,
|
|
@@ -1482,6 +1581,7 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1482
1581
|
queryKeyNorm: inf.attention.queryKeyNorm,
|
|
1483
1582
|
valueNorm: inf.attention.valueNorm,
|
|
1484
1583
|
attentionOutputGate: inf.attention.attentionOutputGate === true,
|
|
1584
|
+
outputGateType: inf.attention.outputGateType ?? null,
|
|
1485
1585
|
queryPreAttnScalar,
|
|
1486
1586
|
layerPipeline: inf.pipeline ?? null,
|
|
1487
1587
|
chatTemplateType,
|
|
@@ -1489,6 +1589,8 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1489
1589
|
chatTemplateThinking,
|
|
1490
1590
|
decodeStrategy,
|
|
1491
1591
|
perLayerInputsSession,
|
|
1592
|
+
sessionSettings,
|
|
1593
|
+
largeWeightsConfig,
|
|
1492
1594
|
kernelPath: null,
|
|
1493
1595
|
visionConfig: resolveVisionConfig(config, manifest),
|
|
1494
1596
|
audioConfig: resolveAudioConfig(config, manifest),
|
|
@@ -2,30 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
import { readBuffer } from '../../../../memory/buffer-pool.js';
|
|
4
4
|
import { isBufferStatsEnabled } from './config.js';
|
|
5
|
+
import { f16ToF32 } from '../../../../loader/dtype-utils.js';
|
|
5
6
|
|
|
6
|
-
|
|
7
|
-
// Math Helpers
|
|
8
|
-
// ============================================================================
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
export function f16ToF32(h) {
|
|
12
|
-
const sign = (h >> 15) & 0x1;
|
|
13
|
-
const exp = (h >> 10) & 0x1f;
|
|
14
|
-
const mant = h & 0x3ff;
|
|
15
|
-
|
|
16
|
-
if (exp === 0) {
|
|
17
|
-
if (mant === 0) return sign ? -0 : 0;
|
|
18
|
-
const f = mant / 1024 * Math.pow(2, -14);
|
|
19
|
-
return sign ? -f : f;
|
|
20
|
-
}
|
|
21
|
-
if (exp === 31) {
|
|
22
|
-
return mant ? NaN : (sign ? -Infinity : Infinity);
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
const f = (1 + mant / 1024) * Math.pow(2, exp - 15);
|
|
26
|
-
return sign ? -f : f;
|
|
27
|
-
}
|
|
28
|
-
|
|
7
|
+
export { f16ToF32 };
|
|
29
8
|
|
|
30
9
|
export function decodeReadback(buffer, dtype) {
|
|
31
10
|
if (dtype === 'f32') {
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// Provides tolerance lookup for first-divergence detection.
|
|
6
6
|
// Models propagation bounds to prevent unbounded drift amplification.
|
|
7
7
|
|
|
8
|
-
import driftPoliciesConfig from '../../../config/
|
|
8
|
+
import driftPoliciesConfig from '../../../config/diagnostics/drift-policies.json' with { type: 'json' };
|
|
9
9
|
|
|
10
10
|
// ============================================================================
|
|
11
11
|
// Policy Lookup
|
|
@@ -36,37 +36,32 @@ export interface ValidationResult {
|
|
|
36
36
|
infCount: number;
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
/**
|
|
40
|
-
* Record scale operation (batched, no submit)
|
|
41
|
-
*/
|
|
42
|
-
export function recordScale(
|
|
43
|
-
recorder: CommandRecorder,
|
|
44
|
-
inputBuffer: GPUBuffer,
|
|
45
|
-
scale: number,
|
|
46
|
-
count: number,
|
|
47
|
-
useF16?: boolean
|
|
48
|
-
): GPUBuffer;
|
|
49
|
-
|
|
50
|
-
/**
|
|
51
|
-
* Scale GPU buffer (standalone, with submit)
|
|
52
|
-
* @deprecated Use recordScale with CommandRecorder instead
|
|
53
|
-
*/
|
|
54
|
-
export function scaleGPUBuffer(
|
|
55
|
-
inputBuffer: GPUBuffer,
|
|
56
|
-
scale: number,
|
|
57
|
-
count: number,
|
|
58
|
-
useF16?: boolean
|
|
59
|
-
): Promise<GPUBuffer>;
|
|
60
|
-
|
|
61
39
|
export function embed(
|
|
62
40
|
tokenIds: number[] | Uint32Array | GPUBuffer,
|
|
63
41
|
embedBuffer: GPUBuffer | Float32Array | CpuWeightBuffer,
|
|
64
42
|
config: EmbedConfig
|
|
65
43
|
): Promise<Tensor>;
|
|
66
44
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
45
|
+
/**
|
|
46
|
+
* True when `value` looks like a range-backed CPU embedding source
|
|
47
|
+
* (has a typed byte range that the loader can pull from).
|
|
48
|
+
*/
|
|
49
|
+
export declare function isRangeBackedCpuEmbeddingSource(value: unknown): boolean;
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Normalize a range bytes payload (typed array or buffer view) into a
|
|
53
|
+
* Uint8Array, throwing a labeled error when the shape is not usable.
|
|
54
|
+
*/
|
|
55
|
+
export declare function normalizeRangeBytes(value: unknown, label: string): Uint8Array;
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Decode a range-backed chunk into `output` at `dstOffset` using the
|
|
59
|
+
* declared `sourceDtype` and per-row `hiddenSize`.
|
|
60
|
+
*/
|
|
61
|
+
export declare function decodeRangeChunkIntoOutput(
|
|
62
|
+
bytes: Uint8Array,
|
|
63
|
+
sourceDtype: string,
|
|
64
|
+
output: Float32Array,
|
|
65
|
+
dstOffset: number,
|
|
71
66
|
hiddenSize: number
|
|
72
|
-
):
|
|
67
|
+
): void;
|
|
@@ -448,42 +448,3 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
448
448
|
|
|
449
449
|
return createTensor(scaledBuffer, dtype, [numTokens, hiddenSize], 'embed_output');
|
|
450
450
|
}
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
export async function validateEmbedding(buffer, label, numTokens, hiddenSize) {
|
|
454
|
-
const device = getDevice();
|
|
455
|
-
if (!device) return null;
|
|
456
|
-
|
|
457
|
-
const sampleSize = Math.min(1024 * 4, buffer.size);
|
|
458
|
-
const sample = await readBuffer(buffer, sampleSize);
|
|
459
|
-
const f32 = new Float32Array(sample);
|
|
460
|
-
|
|
461
|
-
let min = Infinity;
|
|
462
|
-
let max = -Infinity;
|
|
463
|
-
let sum = 0;
|
|
464
|
-
let zeros = 0;
|
|
465
|
-
let nanCount = 0;
|
|
466
|
-
let infCount = 0;
|
|
467
|
-
|
|
468
|
-
for (let i = 0; i < f32.length; i++) {
|
|
469
|
-
const v = f32[i];
|
|
470
|
-
if (Number.isNaN(v)) {
|
|
471
|
-
nanCount++;
|
|
472
|
-
} else if (!Number.isFinite(v)) {
|
|
473
|
-
infCount++;
|
|
474
|
-
} else {
|
|
475
|
-
if (v === 0) zeros++;
|
|
476
|
-
if (v < min) min = v;
|
|
477
|
-
if (v > max) max = v;
|
|
478
|
-
sum += v;
|
|
479
|
-
}
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
const mean = sum / f32.length;
|
|
483
|
-
|
|
484
|
-
trace.embed(`${label}: tokens=${numTokens}, hidden=${hiddenSize}`);
|
|
485
|
-
trace.embed(`${label}: min=${min.toFixed(4)}, max=${max.toFixed(4)}, mean=${mean.toFixed(4)}`);
|
|
486
|
-
trace.embed(`${label}: zeros=${zeros}/${f32.length}, NaN=${nanCount}, Inf=${infCount}`);
|
|
487
|
-
|
|
488
|
-
return { min, max, mean, zeros, nanCount, infCount };
|
|
489
|
-
}
|
|
@@ -2,9 +2,6 @@ import type { KernelPathSchema } from '../../../config/schema/index.js';
|
|
|
2
2
|
import type { RuntimeConfigSchema } from '../../../config/schema/index.js';
|
|
3
3
|
import type { KernelPathSource } from '../../../config/kernel-path-loader.js';
|
|
4
4
|
|
|
5
|
-
export const PRIMARY_EXECUTION_PLAN_ID: 'primary';
|
|
6
|
-
export const FINITENESS_FALLBACK_EXECUTION_PLAN_ID: 'finiteness_fallback';
|
|
7
|
-
|
|
8
5
|
export interface StaticExecutionPlan {
|
|
9
6
|
id: 'primary' | 'finiteness_fallback';
|
|
10
7
|
source: string;
|
|
@@ -127,6 +124,11 @@ export declare function resolveMaxBatchDecodeTokens(config: {
|
|
|
127
124
|
hasLinearAttentionLayers?: boolean;
|
|
128
125
|
}): number | null;
|
|
129
126
|
|
|
127
|
+
export declare function resolvePrefillRecorderChunkLayers(config: {
|
|
128
|
+
hasGpuSplitPerLayerInputs?: boolean;
|
|
129
|
+
numTokens: number;
|
|
130
|
+
}): number;
|
|
131
|
+
|
|
130
132
|
export declare function isDecodeRecorderEnabled(config: {
|
|
131
133
|
hasDevice: boolean;
|
|
132
134
|
debug: boolean;
|
|
@@ -6,8 +6,8 @@ import {
|
|
|
6
6
|
} from './finiteness-policy.js';
|
|
7
7
|
import { log } from '../../../debug/index.js';
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
const PRIMARY_EXECUTION_PLAN_ID = 'primary';
|
|
10
|
+
const FINITENESS_FALLBACK_EXECUTION_PLAN_ID = 'finiteness_fallback';
|
|
11
11
|
const DEFAULT_MAX_TOKENS = 256;
|
|
12
12
|
|
|
13
13
|
function assertOptionalBoolean(value, label) {
|
|
@@ -444,6 +444,19 @@ export function resolveMaxBatchDecodeTokens(config) {
|
|
|
444
444
|
return value;
|
|
445
445
|
}
|
|
446
446
|
|
|
447
|
+
export function resolvePrefillRecorderChunkLayers(config) {
|
|
448
|
+
const value = selectRuleValue('inference', 'execution', 'prefillRecorderChunkLayers', {
|
|
449
|
+
hasGpuSplitPerLayerInputs: config.hasGpuSplitPerLayerInputs === true,
|
|
450
|
+
numTokens: config.numTokens,
|
|
451
|
+
});
|
|
452
|
+
if (!Number.isInteger(value) || value < 1) {
|
|
453
|
+
throw new Error(
|
|
454
|
+
`[ExecutionPlan] prefillRecorderChunkLayers must be a positive integer; got ${JSON.stringify(value)}.`
|
|
455
|
+
);
|
|
456
|
+
}
|
|
457
|
+
return value;
|
|
458
|
+
}
|
|
459
|
+
|
|
447
460
|
export function isDecodeRecorderEnabled(config) {
|
|
448
461
|
return selectRuleValue('inference', 'execution', 'decodeRecorderEnabled', {
|
|
449
462
|
hasDevice: config.hasDevice === true,
|
|
@@ -8,10 +8,6 @@ export declare function requireSessionActivationDtype(
|
|
|
8
8
|
label?: string
|
|
9
9
|
): 'f16' | 'f32';
|
|
10
10
|
|
|
11
|
-
export declare function resolveFinitenessFallbackKernelPathId(
|
|
12
|
-
kernelPathId: string | null | undefined
|
|
13
|
-
): string | null;
|
|
14
|
-
|
|
15
11
|
export declare function buildInlineKernelPath(
|
|
16
12
|
steps: readonly Record<string, unknown>[],
|
|
17
13
|
session: Record<string, unknown> | null,
|
|
@@ -168,17 +168,6 @@ export function assertKernelPathSessionCompatibility(path, session) {
|
|
|
168
168
|
}
|
|
169
169
|
}
|
|
170
170
|
|
|
171
|
-
export function resolveFinitenessFallbackKernelPathId(kernelPathId) {
|
|
172
|
-
return kernelPathId
|
|
173
|
-
? selectRuleValue(
|
|
174
|
-
'inference',
|
|
175
|
-
'kernelPath',
|
|
176
|
-
'finitenessFallback',
|
|
177
|
-
{ kernelPathId }
|
|
178
|
-
)
|
|
179
|
-
: null;
|
|
180
|
-
}
|
|
181
|
-
|
|
182
171
|
export function buildInlineKernelPath(
|
|
183
172
|
steps,
|
|
184
173
|
session,
|
|
@@ -6,6 +6,23 @@ import type {
|
|
|
6
6
|
} from '../../../config/schema/execution-v1.schema.js';
|
|
7
7
|
import type { KernelPathSchema } from '../../../config/schema/index.js';
|
|
8
8
|
|
|
9
|
+
export interface ExecutionV1LaneIntegrity {
|
|
10
|
+
declared: {
|
|
11
|
+
activationDtype: string | null;
|
|
12
|
+
mathDtype: string | null;
|
|
13
|
+
accumDtype: string | null;
|
|
14
|
+
kvDtype: string | null;
|
|
15
|
+
};
|
|
16
|
+
executed: {
|
|
17
|
+
activationDtype: string | null;
|
|
18
|
+
mathDtype: string | null;
|
|
19
|
+
accumDtype: string | null;
|
|
20
|
+
kvDtype: string | null;
|
|
21
|
+
};
|
|
22
|
+
status: 'matches' | 'transformed';
|
|
23
|
+
transforms: string[];
|
|
24
|
+
}
|
|
25
|
+
|
|
9
26
|
export interface ExecutionV1CompiledState {
|
|
10
27
|
session: ExecutionV1SessionSchema;
|
|
11
28
|
policies: ExecutionV1PoliciesSchema;
|
|
@@ -16,6 +33,7 @@ export interface ExecutionV1CompiledState {
|
|
|
16
33
|
};
|
|
17
34
|
runtimeInferencePatch: Record<string, unknown>;
|
|
18
35
|
appliedTransforms?: string[];
|
|
36
|
+
laneIntegrity?: ExecutionV1LaneIntegrity;
|
|
19
37
|
fallbackKernelPath?: KernelPathSchema | null;
|
|
20
38
|
}
|
|
21
39
|
|
|
@@ -18,12 +18,20 @@ import { composeTransforms } from '../../../config/transforms/execution-graph-tr
|
|
|
18
18
|
import { resolveRangeAwareSelectiveWideningConfig } from './finiteness-policy.js';
|
|
19
19
|
import { log } from '../../../debug/index.js';
|
|
20
20
|
|
|
21
|
+
const SESSION_CAPABILITY_TRANSFORMS = new Set([
|
|
22
|
+
'disableRetainQ4KMaterialization',
|
|
23
|
+
]);
|
|
24
|
+
|
|
21
25
|
export function hasExecutionV1(manifestInference) {
|
|
22
26
|
return manifestInference?.schema === EXECUTION_V1_SCHEMA_ID
|
|
23
27
|
&& manifestInference?.execution
|
|
24
28
|
&& typeof manifestInference.execution.kernels === 'object';
|
|
25
29
|
}
|
|
26
30
|
|
|
31
|
+
function isSessionCapabilityTransform(name) {
|
|
32
|
+
return SESSION_CAPABILITY_TRANSFORMS.has(name);
|
|
33
|
+
}
|
|
34
|
+
|
|
27
35
|
function mergeExecutionV1Session(manifestSession, runtimeSession) {
|
|
28
36
|
return mergeRuntimeValues(
|
|
29
37
|
manifestSession ?? {},
|
|
@@ -386,6 +394,8 @@ export function compileExecutionV1(options = {}) {
|
|
|
386
394
|
}
|
|
387
395
|
|
|
388
396
|
const activationDtype = session.compute.defaults.activationDtype;
|
|
397
|
+
const mathDtype = session.compute.defaults.mathDtype ?? null;
|
|
398
|
+
const accumDtype = session.compute.defaults.accumDtype ?? null;
|
|
389
399
|
const kvDtype = session?.kvcache?.kvDtype ?? activationDtype;
|
|
390
400
|
const layerTypes = manifestInference?.layerPattern?.layerTypes ?? null;
|
|
391
401
|
const finitenessPolicy = resolveRangeAwareSelectiveWideningConfig(runtimeCompute);
|
|
@@ -404,10 +414,13 @@ export function compileExecutionV1(options = {}) {
|
|
|
404
414
|
if (capabilities) {
|
|
405
415
|
const graphContext = {
|
|
406
416
|
activationDtype,
|
|
417
|
+
mathDtype,
|
|
418
|
+
accumDtype,
|
|
407
419
|
kvDtype,
|
|
408
420
|
headDim,
|
|
409
421
|
modelId,
|
|
410
422
|
layerTypes,
|
|
423
|
+
retainQ4KMaterialization: session?.retainQ4KMaterialization === true,
|
|
411
424
|
...summarizeExecutionGraphContext(execution),
|
|
412
425
|
};
|
|
413
426
|
const resolved = resolveCapabilityTransforms(capabilities, platform, graphContext);
|
|
@@ -424,22 +437,35 @@ export function compileExecutionV1(options = {}) {
|
|
|
424
437
|
'Use capability-aware remap for manifest-owned execution graphs or choose a compatible runtime.'
|
|
425
438
|
);
|
|
426
439
|
}
|
|
427
|
-
const
|
|
428
|
-
|
|
429
|
-
capabilities,
|
|
430
|
-
platform: platform ?? { id: 'unknown', vendor: 'unknown', architecture: 'unknown' },
|
|
431
|
-
activationDtype,
|
|
432
|
-
kvDtype,
|
|
433
|
-
modelId,
|
|
434
|
-
layerTypes,
|
|
440
|
+
const graphTransforms = resolved.transforms.filter((_, index) => {
|
|
441
|
+
return !isSessionCapabilityTransform(resolved.names[index]);
|
|
435
442
|
});
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
443
|
+
const sessionTransformNames = resolved.names.filter(isSessionCapabilityTransform);
|
|
444
|
+
if (graphTransforms.length > 0) {
|
|
445
|
+
const composed = composeTransforms(...graphTransforms);
|
|
446
|
+
const transformed = composed(execution, {
|
|
447
|
+
capabilities,
|
|
448
|
+
platform: platform ?? { id: 'unknown', vendor: 'unknown', architecture: 'unknown' },
|
|
449
|
+
activationDtype,
|
|
450
|
+
mathDtype,
|
|
451
|
+
accumDtype,
|
|
452
|
+
kvDtype,
|
|
453
|
+
modelId,
|
|
454
|
+
layerTypes,
|
|
455
|
+
});
|
|
456
|
+
if (transformed !== execution) {
|
|
457
|
+
execution = transformed;
|
|
458
|
+
graphWasTransformed = true;
|
|
459
|
+
appliedTransformNames.push(...resolved.names.filter((name) => !isSessionCapabilityTransform(name)));
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
if (sessionTransformNames.length > 0) {
|
|
463
|
+
appliedTransformNames.push(...sessionTransformNames);
|
|
464
|
+
}
|
|
465
|
+
if (appliedTransformNames.length > 0) {
|
|
440
466
|
log.info(
|
|
441
467
|
'ExecutionV1',
|
|
442
|
-
`Capability transforms applied: [${
|
|
468
|
+
`Capability transforms applied: [${appliedTransformNames.join(', ')}] (${resolved.reason})`
|
|
443
469
|
);
|
|
444
470
|
}
|
|
445
471
|
} else {
|
|
@@ -456,6 +482,8 @@ export function compileExecutionV1(options = {}) {
|
|
|
456
482
|
capabilities,
|
|
457
483
|
platform: platform ?? { id: 'unknown', vendor: 'unknown', architecture: 'unknown' },
|
|
458
484
|
activationDtype,
|
|
485
|
+
mathDtype,
|
|
486
|
+
accumDtype,
|
|
459
487
|
kvDtype,
|
|
460
488
|
headDim,
|
|
461
489
|
modelId,
|
|
@@ -486,25 +514,33 @@ export function compileExecutionV1(options = {}) {
|
|
|
486
514
|
// KV cache and all compute dtypes must also be f32.
|
|
487
515
|
const activationWidened = appliedTransformNames.includes('widenToF32Activations');
|
|
488
516
|
const fullF32 = activationWidened && capabilities?.hasF16 === false;
|
|
489
|
-
const
|
|
490
|
-
|
|
517
|
+
const useSelectiveF16Primary =
|
|
518
|
+
(
|
|
519
|
+
appliedTransformNames.includes('useQwenF16PrimaryMatmuls')
|
|
520
|
+
|| appliedTransformNames.includes('useGemma4Int4PleSelectiveF16Decode')
|
|
521
|
+
)
|
|
491
522
|
&& !appliedTransformNames.includes('narrowToF16Activations');
|
|
523
|
+
const retainQ4KMaterializationDisabled =
|
|
524
|
+
appliedTransformNames.includes('disableRetainQ4KMaterialization');
|
|
492
525
|
let effectiveSession = session;
|
|
493
|
-
if (activationWidened ||
|
|
526
|
+
if (activationWidened || useSelectiveF16Primary || retainQ4KMaterializationDisabled) {
|
|
494
527
|
const f32Defaults = fullF32
|
|
495
528
|
? { activationDtype: 'f32', mathDtype: 'f32', accumDtype: 'f32', outputDtype: 'f32' }
|
|
496
|
-
:
|
|
529
|
+
: useSelectiveF16Primary
|
|
497
530
|
? { activationDtype: 'f32', outputDtype: 'f32' }
|
|
498
531
|
: { activationDtype: 'f32' };
|
|
499
532
|
effectiveSession = {
|
|
500
533
|
...session,
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
...session.compute
|
|
505
|
-
|
|
534
|
+
...(retainQ4KMaterializationDisabled ? { retainQ4KMaterialization: false } : {}),
|
|
535
|
+
...(activationWidened || useSelectiveF16Primary ? {
|
|
536
|
+
compute: {
|
|
537
|
+
...session.compute,
|
|
538
|
+
defaults: {
|
|
539
|
+
...session.compute.defaults,
|
|
540
|
+
...f32Defaults,
|
|
541
|
+
},
|
|
506
542
|
},
|
|
507
|
-
},
|
|
543
|
+
} : {}),
|
|
508
544
|
...(fullF32 && session?.kvcache ? {
|
|
509
545
|
kvcache: {
|
|
510
546
|
...session.kvcache,
|
|
@@ -570,6 +606,37 @@ export function compileExecutionV1(options = {}) {
|
|
|
570
606
|
}
|
|
571
607
|
}
|
|
572
608
|
|
|
609
|
+
// Lane integrity: capture the pre-transform (manifest+profile-agreed) lane
|
|
610
|
+
// and the post-transform (actually dispatched) lane so receipts can honestly
|
|
611
|
+
// surface a capability-driven widening. The manifest-binding gate ensures
|
|
612
|
+
// the pre-transform session already matches manifest quantizationInfo.compute;
|
|
613
|
+
// any drift here is owned by capability transforms (e.g., widenToF32Activations
|
|
614
|
+
// on hasF16=false GPUs).
|
|
615
|
+
const declaredLane = {
|
|
616
|
+
activationDtype,
|
|
617
|
+
mathDtype,
|
|
618
|
+
accumDtype,
|
|
619
|
+
kvDtype,
|
|
620
|
+
};
|
|
621
|
+
const executedLane = {
|
|
622
|
+
activationDtype: effectiveSession.compute?.defaults?.activationDtype ?? null,
|
|
623
|
+
mathDtype: effectiveSession.compute?.defaults?.mathDtype ?? null,
|
|
624
|
+
accumDtype: effectiveSession.compute?.defaults?.accumDtype ?? null,
|
|
625
|
+
kvDtype: effectiveSession.kvcache?.kvDtype ?? effectiveSession.compute?.defaults?.activationDtype ?? null,
|
|
626
|
+
};
|
|
627
|
+
const laneFieldDelta = (
|
|
628
|
+
declaredLane.activationDtype !== executedLane.activationDtype
|
|
629
|
+
|| declaredLane.mathDtype !== executedLane.mathDtype
|
|
630
|
+
|| declaredLane.accumDtype !== executedLane.accumDtype
|
|
631
|
+
|| declaredLane.kvDtype !== executedLane.kvDtype
|
|
632
|
+
);
|
|
633
|
+
const laneIntegrity = {
|
|
634
|
+
declared: declaredLane,
|
|
635
|
+
executed: executedLane,
|
|
636
|
+
status: laneFieldDelta ? 'transformed' : 'matches',
|
|
637
|
+
transforms: [...appliedTransformNames],
|
|
638
|
+
};
|
|
639
|
+
|
|
573
640
|
const layerPipelineResult = buildLayerPipelineFromExecution(resolvedSteps, {
|
|
574
641
|
logIncompatibleOps: !(kernelPath && inlineKernelPathEnabled),
|
|
575
642
|
ffnDtypeFallback: requireSessionActivationDtype(effectiveSession),
|
|
@@ -600,6 +667,7 @@ export function compileExecutionV1(options = {}) {
|
|
|
600
667
|
...(layerPipeline ? { pipeline: layerPipeline } : {}),
|
|
601
668
|
},
|
|
602
669
|
appliedTransforms: appliedTransformNames,
|
|
670
|
+
laneIntegrity,
|
|
603
671
|
fallbackKernelPath,
|
|
604
672
|
};
|
|
605
673
|
}
|
|
@@ -621,6 +689,7 @@ export function compileExecutionV1(options = {}) {
|
|
|
621
689
|
export function applyExecutionV1RuntimeConfig(options = {}) {
|
|
622
690
|
const runtimeConfig = options.runtimeConfig ?? null;
|
|
623
691
|
const runtimeOverrides = options.runtimeOverrides;
|
|
692
|
+
const hasExplicitRuntimeOverrides = runtimeOverrides != null;
|
|
624
693
|
const manifest = options.manifest ?? null;
|
|
625
694
|
if (!runtimeConfig || !manifest?.inference) {
|
|
626
695
|
return { runtimeConfig, executionV1State: null };
|
|
@@ -644,10 +713,10 @@ export function applyExecutionV1RuntimeConfig(options = {}) {
|
|
|
644
713
|
headDim: options.headDim ?? manifest.architecture?.headDim ?? null,
|
|
645
714
|
capabilities: options.capabilities ?? null,
|
|
646
715
|
platform: options.platform ?? null,
|
|
647
|
-
runtimeSession:
|
|
716
|
+
runtimeSession: hasExplicitRuntimeOverrides
|
|
648
717
|
? resolveRuntimeInferenceOverrideSection(runtimeOverrides, 'session')
|
|
649
718
|
: (runtimeConfig.inference?.session ?? null),
|
|
650
|
-
runtimeCompute:
|
|
719
|
+
runtimeCompute: hasExplicitRuntimeOverrides
|
|
651
720
|
? resolveRuntimeInferenceOverrideSection(runtimeOverrides, 'compute')
|
|
652
721
|
: (runtimeConfig.inference?.compute ?? null),
|
|
653
722
|
kernelPathPolicy: runtimeConfig.inference?.kernelPathPolicy ?? null,
|