doppler-gpu 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +19 -15
- package/package.json +77 -29
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +64 -6
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +524 -350
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +12 -2
- package/src/config/schema/doppler.schema.js +108 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +68 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +76 -0
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +107 -0
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +3 -21
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +6 -5
- package/src/gpu/kernels/index.js +3 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +12 -0
- package/src/inference/browser-harness-text-helpers.js +223 -14
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +264 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +22 -27
- package/src/inference/pipelines/text/embed.js +0 -39
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +4 -5
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +8 -3
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +223 -547
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +5 -2
- package/src/inference/pipelines/text/init.js +183 -724
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.js +2 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +120 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +223 -27
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +13 -3
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +11 -3
- package/src/storage/artifact-storage-context.js +231 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +21 -24
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +2 -0
- package/src/tooling/source-runtime-bundle.js +3 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { cloneJsonValue as cloneConfigTree } from '../../utils/clone-json.js';
|
|
1
2
|
import { DEFAULT_LOADING_CONFIG } from './loading.schema.js';
|
|
2
3
|
import { DEFAULT_SHARED_RUNTIME_CONFIG } from './shared-runtime.schema.js';
|
|
3
4
|
import { DEFAULT_EMULATION_CONFIG, createEmulationConfig } from './emulation.schema.js';
|
|
@@ -29,6 +30,7 @@ export const DEFAULT_LARGE_WEIGHT_CONFIG = Object.freeze({
|
|
|
29
30
|
safetyRatio: 0.9,
|
|
30
31
|
preferF16: true,
|
|
31
32
|
lmHeadChunkRows: null,
|
|
33
|
+
gpuResidentOverrides: null,
|
|
32
34
|
});
|
|
33
35
|
|
|
34
36
|
export const DEFAULT_RUNTIME_CONFIG = {
|
|
@@ -55,9 +57,85 @@ export const DEFAULT_RUNTIME_CONFIG = {
|
|
|
55
57
|
generation: {
|
|
56
58
|
disableMultiTokenDecode: false,
|
|
57
59
|
},
|
|
60
|
+
kernelPathPolicy: mergeKernelPathPolicy(undefined, {
|
|
61
|
+
mode: 'capability-aware',
|
|
62
|
+
sourceScope: ['model', 'manifest', 'config'],
|
|
63
|
+
onIncompatible: 'remap',
|
|
64
|
+
}),
|
|
58
65
|
chatTemplate: DEFAULT_CHAT_TEMPLATE_CONFIG,
|
|
59
66
|
session: {
|
|
60
67
|
kvcache: DEFAULT_KVCACHE_CONFIG,
|
|
68
|
+
// "sync" (default): wait for each prefill chunk to finish on GPU before
|
|
69
|
+
// recording the next. Required when profile timings are being
|
|
70
|
+
// resolved and when low-memory constraints demand deterministic
|
|
71
|
+
// buffer release per chunk.
|
|
72
|
+
// "async": queue each prefill chunk without waiting — GPU queue ordering
|
|
73
|
+
// keeps commands correct and deferred cleanup releases tracked
|
|
74
|
+
// buffers when work completes. Reduces CPU-GPU roundtrips; profile
|
|
75
|
+
// paths override to sync automatically.
|
|
76
|
+
prefillChunkSubmitMode: 'sync',
|
|
77
|
+
// Number of transformer layers recorded per prefill command recorder
|
|
78
|
+
// before a chunk boundary fires. Larger values → fewer chunk
|
|
79
|
+
// transitions but longer intermediate-buffer lifetime. Default 4 is
|
|
80
|
+
// memory-conservative; profiles with known-safe memory can raise it.
|
|
81
|
+
prefillChunkLayers: 4,
|
|
82
|
+
// Opt into the flash-attention prefill kernel (head_dim=256, f16 KV,
|
|
83
|
+
// contiguous layout). Two-pass dispatch raises RDNA3 workgroup
|
|
84
|
+
// occupancy by KV-axis splitting. The kernel itself enforces head_dim
|
|
85
|
+
// and layout preconditions at runtime. Default false until parity is
|
|
86
|
+
// validated on target hardware.
|
|
87
|
+
useFlashPrefillAttention: false,
|
|
88
|
+
// Opt into the fused gate + up projection + GeGLU activation prefill
|
|
89
|
+
// kernel (replaces gate_proj + up_proj + gelu activation dispatches
|
|
90
|
+
// with a single fused compute pass). Requires f16 weights + gelu
|
|
91
|
+
// activation + no LoRA on gate/up. Default false until parity is
|
|
92
|
+
// validated on target hardware.
|
|
93
|
+
useFusedGateUpGelu: false,
|
|
94
|
+
// Opt into the register-tiled Q4_K prefill matmul kernel (64x64 outputs
|
|
95
|
+
// per workgroup via 4x4 register tile per thread). Amortizes Q4_K
|
|
96
|
+
// dequantization across 16x the outputs of q4_fused_batched_f16a,
|
|
97
|
+
// replacing ~1024x too many launched workgroups on prefill shapes.
|
|
98
|
+
// Applied only to Q4_K-weight matmuls with f16 activations, f16 output,
|
|
99
|
+
// hasSubgroups, M>=16. Default false until parity is validated.
|
|
100
|
+
useTiledQ4KPrefill: false,
|
|
101
|
+
// Force the loader to retain Q4_K packed weights alongside the
|
|
102
|
+
// dequantized dense buffer ("mixed" materialization mode). Without a
|
|
103
|
+
// fused projection kernel declared in the execution graph, the default
|
|
104
|
+
// is "dense" which drops the Q4_K buffer after dequant — preventing the
|
|
105
|
+
// FFN fusion rule from firing its `hasQ4KMaterialization=true` branch.
|
|
106
|
+
// Memory cost: ~50% extra weight storage for Q4_K matmul weights.
|
|
107
|
+
retainQ4KMaterialization: false,
|
|
108
|
+
// Opt into the WideTile Q4_K prefill matmul (adapted from ORT
|
|
109
|
+
// MatMulNBitsWideTile). Register-tiled accumulation, 1 thread per
|
|
110
|
+
// output column, TILE_M rows accumulated simultaneously. Dramatically
|
|
111
|
+
// fewer workgroups launched vs q4_fused_batched_f16 at prefill shapes.
|
|
112
|
+
// Requires f32 activations + Q4_K weights + shader-f16 + M >= TILE_M.
|
|
113
|
+
// Default false until correctness + perf parity validated on target
|
|
114
|
+
// hardware.
|
|
115
|
+
useWideTileQ4KPrefill: false,
|
|
116
|
+
// Opt into the single-pass flash attention prefill kernel adapted from
|
|
117
|
+
// ORT's flash_attention.wgsl.template. 64 threads = 64 queries per WG;
|
|
118
|
+
// private Q/O tiles, shared K/V tiles, online softmax, no reduce pass.
|
|
119
|
+
// At Gemma 4 E2B prefill=64 this dispatches 8 WGs total (vs ~32 for the
|
|
120
|
+
// split+reduce Doppler flash path). Requires head_dim=256, f16 KV,
|
|
121
|
+
// contiguous KV layout, seqLen > 1. Supersedes useFlashPrefillAttention
|
|
122
|
+
// when both flags are set (this path is picked first). Default false.
|
|
123
|
+
useOrtFlashPrefillAttention: false,
|
|
124
|
+
// Opt into the WideTile Q4_K matmul + residual fusion at the ffn_down
|
|
125
|
+
// + ffn_residual call site (dense.js). Eliminates one separate residual
|
|
126
|
+
// dispatch per layer (~0.85 ms bubble × 35 layers ≈ 30 ms prefill
|
|
127
|
+
// savings). Requires f32 activations + Q4_K weights retained + prefill.
|
|
128
|
+
// Default false until correctness + perf validated.
|
|
129
|
+
useWideTileResidualFusion: false,
|
|
130
|
+
// Opt into the RMSNorm + WideTile Q4_K matmul fusion at pre-matmul
|
|
131
|
+
// norm sites (input_norm→q/k/v_proj, pre_feedforward_norm→gate/up).
|
|
132
|
+
// Each fused call recomputes RMS internally (redundant across q/k/v
|
|
133
|
+
// but negligible) and skips the standalone rmsnorm dispatch upstream.
|
|
134
|
+
// Saves ~2 dispatches/layer × 35 layers = 70 dispatches × ~0.85 ms
|
|
135
|
+
// bubble ≈ 60 ms prefill savings. Requires f32 activations + Q4_K
|
|
136
|
+
// weights retained + prefill. Default false until wiring lands and
|
|
137
|
+
// correctness validates.
|
|
138
|
+
useFusedRmsnormWideTile: false,
|
|
61
139
|
},
|
|
62
140
|
executionPatch: {},
|
|
63
141
|
},
|
|
@@ -73,13 +151,6 @@ export const DEFAULT_DOPPLER_CONFIG = {
|
|
|
73
151
|
runtime: DEFAULT_RUNTIME_CONFIG,
|
|
74
152
|
};
|
|
75
153
|
|
|
76
|
-
function cloneConfigTree(value) {
|
|
77
|
-
if (typeof structuredClone === 'function') {
|
|
78
|
-
return structuredClone(value);
|
|
79
|
-
}
|
|
80
|
-
return JSON.parse(JSON.stringify(value));
|
|
81
|
-
}
|
|
82
|
-
|
|
83
154
|
// =============================================================================
|
|
84
155
|
// Factory Function
|
|
85
156
|
// =============================================================================
|
|
@@ -304,7 +375,7 @@ function mergeInferenceConfig(
|
|
|
304
375
|
speculative: { ...base.speculative, ...overrides.speculative },
|
|
305
376
|
generation: { ...base.generation, ...overrides.generation },
|
|
306
377
|
pipeline: overrides.pipeline ?? base.pipeline,
|
|
307
|
-
kernelPath:
|
|
378
|
+
kernelPath: mergeRuntimeKernelPath(base.kernelPath, overrides.kernelPath),
|
|
308
379
|
kernelPathSource: overrides.kernelPathSource ?? base.kernelPathSource,
|
|
309
380
|
kernelPathPolicy: mergeKernelPathPolicy(baseKernelPathPolicy, overrideKernelPathPolicy),
|
|
310
381
|
chatTemplate: mergeShallowObject(base.chatTemplate, overrides.chatTemplate),
|
|
@@ -332,6 +403,32 @@ function mergeInferenceConfig(
|
|
|
332
403
|
};
|
|
333
404
|
}
|
|
334
405
|
|
|
406
|
+
function mergeRuntimeKernelPath(
|
|
407
|
+
baseKernelPath,
|
|
408
|
+
overrideKernelPath
|
|
409
|
+
) {
|
|
410
|
+
const kernelPath = chooseDefined(overrideKernelPath, baseKernelPath);
|
|
411
|
+
assertRuntimeKernelPath(kernelPath);
|
|
412
|
+
return kernelPath;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
function assertRuntimeKernelPath(kernelPath) {
|
|
416
|
+
if (kernelPath === undefined || kernelPath === null) {
|
|
417
|
+
return;
|
|
418
|
+
}
|
|
419
|
+
if (typeof kernelPath === 'string') {
|
|
420
|
+
throw new Error(
|
|
421
|
+
'DopplerConfigError: runtime.inference.kernelPath no longer accepts string registry IDs. ' +
|
|
422
|
+
'Use an inline kernel path object generated from execution-v1, or leave kernelPath null.'
|
|
423
|
+
);
|
|
424
|
+
}
|
|
425
|
+
if (typeof kernelPath !== 'object' || Array.isArray(kernelPath)) {
|
|
426
|
+
throw new Error(
|
|
427
|
+
'DopplerConfigError: runtime.inference.kernelPath must be an inline kernel path object or null.'
|
|
428
|
+
);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
335
432
|
function mergeKernelThresholds(
|
|
336
433
|
base,
|
|
337
434
|
overrides
|
|
@@ -362,6 +459,9 @@ function mergeDebugConfig(
|
|
|
362
459
|
logLevel: { ...base.logLevel, ...overrides.logLevel },
|
|
363
460
|
trace: { ...base.trace, ...overrides.trace },
|
|
364
461
|
pipeline: { ...base.pipeline, ...overrides.pipeline },
|
|
462
|
+
loader: { ...base.loader, ...overrides.loader },
|
|
463
|
+
matmul: { ...base.matmul, ...overrides.matmul },
|
|
464
|
+
kernelTrace: { ...base.kernelTrace, ...overrides.kernelTrace },
|
|
365
465
|
probes: overrides.probes ?? base.probes,
|
|
366
466
|
profiler: { ...base.profiler, ...overrides.profiler },
|
|
367
467
|
perfGuards: { ...base.perfGuards, ...overrides.perfGuards },
|
|
@@ -349,3 +349,6 @@ export declare function createEmulationConfig(
|
|
|
349
349
|
* Get chip profile config for a specific chip type
|
|
350
350
|
*/
|
|
351
351
|
export declare function getChipProfile(chipType: EmulatedChipType): Partial<EmulationConfigSchema>;
|
|
352
|
+
|
|
353
|
+
/** Format a byte count as a human-readable string (e.g. "1.5 GB"). */
|
|
354
|
+
export declare function formatBytes(bytes: number): string;
|
|
@@ -10,21 +10,21 @@ const gb200Nvl72Profile = await loadJson('../platforms/nvidia-gb200-nvl72.json',
|
|
|
10
10
|
// GPU Specifications
|
|
11
11
|
// =============================================================================
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
const H100_GPU_SPEC = {
|
|
14
14
|
name: 'H100',
|
|
15
15
|
vramBytes: 96 * GB,
|
|
16
16
|
hbmBandwidthBytesPerSec: 3.35e12, // 3.35 TB/s HBM3
|
|
17
17
|
fp16Tflops: 1979, // ~2 PFLOPS
|
|
18
18
|
};
|
|
19
19
|
|
|
20
|
-
|
|
20
|
+
const H200_GPU_SPEC = {
|
|
21
21
|
name: 'H200',
|
|
22
22
|
vramBytes: 144 * GB,
|
|
23
23
|
hbmBandwidthBytesPerSec: 4.8e12, // 4.8 TB/s HBM3e
|
|
24
24
|
fp16Tflops: 1979, // ~2 PFLOPS
|
|
25
25
|
};
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
const B200_GPU_SPEC = {
|
|
28
28
|
name: 'B200',
|
|
29
29
|
vramBytes: 192 * GB,
|
|
30
30
|
hbmBandwidthBytesPerSec: 8e12, // 8 TB/s HBM3e
|
|
@@ -38,7 +38,7 @@ export const DEFAULT_GH200_GPU_SPEC = H200_GPU_SPEC;
|
|
|
38
38
|
// CPU Specifications
|
|
39
39
|
// =============================================================================
|
|
40
40
|
|
|
41
|
-
|
|
41
|
+
const GRACE_CPU_SPEC = {
|
|
42
42
|
name: 'Grace',
|
|
43
43
|
cores: 72,
|
|
44
44
|
memoryBytes: 480 * GB, // LPDDR5X
|
|
@@ -51,12 +51,12 @@ export const DEFAULT_GH200_CPU_SPEC = GRACE_CPU_SPEC;
|
|
|
51
51
|
// NVLink Specifications
|
|
52
52
|
// =============================================================================
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
const NVLINK_4_SPEC = {
|
|
55
55
|
bandwidthBytesPerSec: 900e9, // 900 GB/s
|
|
56
56
|
latencyUs: 1.0, // ~1 microsecond
|
|
57
57
|
};
|
|
58
58
|
|
|
59
|
-
|
|
59
|
+
const NVLINK_5_SPEC = {
|
|
60
60
|
bandwidthBytesPerSec: 1.8e12, // 1.8 TB/s
|
|
61
61
|
latencyUs: 0.8, // ~0.8 microseconds
|
|
62
62
|
};
|
|
@@ -73,28 +73,28 @@ export const DEFAULT_NVLINK_C2C_SPEC = {
|
|
|
73
73
|
// Cluster Topologies
|
|
74
74
|
// =============================================================================
|
|
75
75
|
|
|
76
|
-
|
|
76
|
+
const GH200_TOPOLOGY = {
|
|
77
77
|
gpuCount: 1,
|
|
78
78
|
gpusPerNode: 1,
|
|
79
79
|
nodeCount: 1,
|
|
80
80
|
cpuCount: 1,
|
|
81
81
|
};
|
|
82
82
|
|
|
83
|
-
|
|
83
|
+
const GH200_NVL2_TOPOLOGY = {
|
|
84
84
|
gpuCount: 2,
|
|
85
85
|
gpusPerNode: 2,
|
|
86
86
|
nodeCount: 1,
|
|
87
87
|
cpuCount: 2,
|
|
88
88
|
};
|
|
89
89
|
|
|
90
|
-
|
|
90
|
+
const GB200_8GPU_TOPOLOGY = {
|
|
91
91
|
gpuCount: 8,
|
|
92
92
|
gpusPerNode: 8,
|
|
93
93
|
nodeCount: 1,
|
|
94
94
|
cpuCount: 2,
|
|
95
95
|
};
|
|
96
96
|
|
|
97
|
-
|
|
97
|
+
const GB200_NVL72_TOPOLOGY = {
|
|
98
98
|
gpuCount: 72,
|
|
99
99
|
gpusPerNode: 8,
|
|
100
100
|
nodeCount: 9,
|
|
@@ -125,7 +125,7 @@ export const DEFAULT_PARALLELISM_CONFIG = {
|
|
|
125
125
|
},
|
|
126
126
|
};
|
|
127
127
|
|
|
128
|
-
|
|
128
|
+
const TP2_PARALLELISM_CONFIG = {
|
|
129
129
|
...DEFAULT_PARALLELISM_CONFIG,
|
|
130
130
|
tensorParallel: {
|
|
131
131
|
enabled: true,
|
|
@@ -133,7 +133,7 @@ export const TP2_PARALLELISM_CONFIG = {
|
|
|
133
133
|
},
|
|
134
134
|
};
|
|
135
135
|
|
|
136
|
-
|
|
136
|
+
const TP8_PARALLELISM_CONFIG = {
|
|
137
137
|
...DEFAULT_PARALLELISM_CONFIG,
|
|
138
138
|
tensorParallel: {
|
|
139
139
|
enabled: true,
|
|
@@ -145,7 +145,7 @@ export const TP8_PARALLELISM_CONFIG = {
|
|
|
145
145
|
// Timing Scaling Defaults
|
|
146
146
|
// =============================================================================
|
|
147
147
|
|
|
148
|
-
|
|
148
|
+
const DEFAULT_TIMING_SCALING = {
|
|
149
149
|
computeScale: 1.0,
|
|
150
150
|
memoryScale: 1.0,
|
|
151
151
|
nvlinkScale: 1.0,
|
|
@@ -273,11 +273,11 @@ function mergeParallelismConfig(base, overrides) {
|
|
|
273
273
|
// Utility Functions
|
|
274
274
|
// =============================================================================
|
|
275
275
|
|
|
276
|
-
|
|
276
|
+
function calculateTotalVram(config) {
|
|
277
277
|
return config.gpuSpec.vramBytes * config.topology.gpuCount;
|
|
278
278
|
}
|
|
279
279
|
|
|
280
|
-
|
|
280
|
+
function calculateTotalCpuMemory(config) {
|
|
281
281
|
return config.cpuSpec.memoryBytes * config.topology.cpuCount;
|
|
282
282
|
}
|
|
283
283
|
|
|
@@ -296,6 +296,6 @@ export function formatBytes(bytes) {
|
|
|
296
296
|
return `${bytes} B`;
|
|
297
297
|
}
|
|
298
298
|
|
|
299
|
-
|
|
299
|
+
function formatBandwidth(bytesPerSec) {
|
|
300
300
|
return `${formatBytes(bytesPerSec)}/s`;
|
|
301
301
|
}
|
|
@@ -173,6 +173,16 @@ export interface ExecutionV1SessionSchema {
|
|
|
173
173
|
decodeLoop: ExecutionV1DecodeLoopSchema | null;
|
|
174
174
|
perLayerInputs: ExecutionV1PerLayerInputsSessionSchema;
|
|
175
175
|
speculation: ExecutionV1SelfSpeculationSchema | null;
|
|
176
|
+
/** "sync" (default): wait each prefill chunk. "async": queue without waiting. */
|
|
177
|
+
prefillChunkSubmitMode: 'sync' | 'async';
|
|
178
|
+
/** Opt into flash-attention prefill kernel. Requires head_dim=256, f16 KV, contiguous layout. */
|
|
179
|
+
useFlashPrefillAttention: boolean;
|
|
180
|
+
/** Opt into WideTile Q4_K prefill matmul. Requires f32 activations + Q4_K weights + shader-f16 + M>=TILE_M. */
|
|
181
|
+
useWideTileQ4KPrefill: boolean;
|
|
182
|
+
/** Retain Q4_K packed weights alongside dense buffer (mixed materialization). ~50% extra Q4_K memory. */
|
|
183
|
+
retainQ4KMaterialization: boolean;
|
|
184
|
+
/** Use f32-accumulator twin of fused-Q4K f16a multicol gemv. Closes the f16-accum decode gap on AMD RDNA3 where f32 accumulation outperforms f16 by ~60% at FFN gemv shape. Gated by capability rule. */
|
|
185
|
+
useF32AccumF16ioMatmul: boolean;
|
|
176
186
|
}
|
|
177
187
|
|
|
178
188
|
// === Policies ===
|
|
@@ -295,12 +305,13 @@ export interface ExecutionV1ExpandedStepSchema {
|
|
|
295
305
|
|
|
296
306
|
export declare const EXECUTION_V1_SCHEMA_ID: string;
|
|
297
307
|
export declare const READBACK_MODES: readonly ['sequential', 'overlapped', 'auto'];
|
|
308
|
+
export declare const PREFILL_CHUNK_SUBMIT_MODES: readonly ['sync', 'async'];
|
|
298
309
|
export declare const PER_LAYER_INPUT_MATERIALIZATION_MODES: readonly ['auto', 'range_backed', 'cpu_resident', 'gpu_resident'];
|
|
299
310
|
export declare const PER_LAYER_INPUT_ROW_CACHE_MODES: readonly ['off', 'lru'];
|
|
300
311
|
export declare const PER_LAYER_INPUT_PREFETCH_MODES: readonly ['off', 'next_token'];
|
|
301
312
|
export declare const PER_LAYER_INPUT_GPU_UPLOAD_MODES: readonly ['per_step_slices', 'per_batch_slices'];
|
|
313
|
+
export declare const PER_LAYER_INPUT_HOT_CACHE_MODES: readonly ['off', 'prepared_tokens', 'tokenizer_scores'];
|
|
302
314
|
export declare const DEFAULT_EXECUTION_V1_COMPUTE_DEFAULTS: ExecutionV1ComputeDefaultsSchema;
|
|
303
|
-
export declare const DEFAULT_EXECUTION_V1_PER_LAYER_INPUTS_SESSION: ExecutionV1PerLayerInputsSessionSchema;
|
|
304
315
|
export declare const DEFAULT_EXECUTION_V1_SESSION: ExecutionV1SessionSchema;
|
|
305
316
|
export declare const DEFAULT_EXECUTION_V1_POLICIES: ExecutionV1PoliciesSchema;
|
|
306
317
|
export declare const DEFAULT_EXECUTION_V1_PATCH: ExecutionV1PatchSchema;
|
|
@@ -15,6 +15,7 @@ export const DEFAULT_EXECUTION_V1_COMPUTE_DEFAULTS = {
|
|
|
15
15
|
};
|
|
16
16
|
|
|
17
17
|
export const READBACK_MODES = Object.freeze(['sequential', 'overlapped', 'auto']);
|
|
18
|
+
export const PREFILL_CHUNK_SUBMIT_MODES = Object.freeze(['sync', 'async']);
|
|
18
19
|
export const PER_LAYER_INPUT_MATERIALIZATION_MODES = Object.freeze([
|
|
19
20
|
'auto',
|
|
20
21
|
'range_backed',
|
|
@@ -34,7 +35,7 @@ export const PER_LAYER_INPUT_HOT_CACHE_MODES = Object.freeze([
|
|
|
34
35
|
'tokenizer_scores',
|
|
35
36
|
]);
|
|
36
37
|
|
|
37
|
-
|
|
38
|
+
const DEFAULT_EXECUTION_V1_PER_LAYER_INPUTS_SESSION = {
|
|
38
39
|
materialization: 'auto',
|
|
39
40
|
rowCache: {
|
|
40
41
|
mode: 'lru',
|
|
@@ -66,6 +67,11 @@ export const DEFAULT_EXECUTION_V1_SESSION = {
|
|
|
66
67
|
decodeLoop: null,
|
|
67
68
|
perLayerInputs: { ...DEFAULT_EXECUTION_V1_PER_LAYER_INPUTS_SESSION },
|
|
68
69
|
speculation: null,
|
|
70
|
+
prefillChunkSubmitMode: 'sync',
|
|
71
|
+
useFlashPrefillAttention: false,
|
|
72
|
+
useWideTileQ4KPrefill: false,
|
|
73
|
+
retainQ4KMaterialization: false,
|
|
74
|
+
useF32AccumF16ioMatmul: false,
|
|
69
75
|
};
|
|
70
76
|
|
|
71
77
|
export const DEFAULT_EXECUTION_V1_POLICIES = {
|
|
@@ -45,6 +45,10 @@ export {
|
|
|
45
45
|
type RuntimeOptimizationsSchema,
|
|
46
46
|
type QuantizationInfoSchema,
|
|
47
47
|
type ConversionInfoSchema,
|
|
48
|
+
type SourceArtifactFormat,
|
|
49
|
+
type ArtifactCompleteness,
|
|
50
|
+
type ManifestArtifactIdentitySchema,
|
|
51
|
+
type ManifestWeightsRefSchema,
|
|
48
52
|
type ManifestSchema,
|
|
49
53
|
type AdapterConfigSchema,
|
|
50
54
|
type ProvenanceSchema,
|
|
@@ -78,7 +82,6 @@ export {
|
|
|
78
82
|
type KernelStepSchema,
|
|
79
83
|
type LayerKernelPathSchema,
|
|
80
84
|
type LayerOverrideSchema,
|
|
81
|
-
type BuiltinKernelPathId,
|
|
82
85
|
DEFAULT_ENTRY,
|
|
83
86
|
DEFAULT_INPUT,
|
|
84
87
|
DEFAULT_OUTPUT,
|
|
@@ -231,6 +234,23 @@ export {
|
|
|
231
234
|
validateBrowserSuiteMetrics,
|
|
232
235
|
} from './browser-suite-metrics.schema.js';
|
|
233
236
|
|
|
237
|
+
// =============================================================================
|
|
238
|
+
// Program Bundle Schema
|
|
239
|
+
// =============================================================================
|
|
240
|
+
export {
|
|
241
|
+
type ProgramBundle,
|
|
242
|
+
type ProgramBundleArtifact,
|
|
243
|
+
type ProgramBundleReferenceTranscript,
|
|
244
|
+
type ProgramBundleWgslModule,
|
|
245
|
+
PROGRAM_BUNDLE_SCHEMA_VERSION,
|
|
246
|
+
PROGRAM_BUNDLE_SCHEMA_ID,
|
|
247
|
+
PROGRAM_BUNDLE_HOST_SCHEMA_ID,
|
|
248
|
+
PROGRAM_BUNDLE_HOST_JS_SUBSET,
|
|
249
|
+
PROGRAM_BUNDLE_CAPTURE_PROFILE_SCHEMA_ID,
|
|
250
|
+
PROGRAM_BUNDLE_REFERENCE_TRANSCRIPT_SCHEMA_ID,
|
|
251
|
+
validateProgramBundle,
|
|
252
|
+
} from './program-bundle.schema.js';
|
|
253
|
+
|
|
234
254
|
// =============================================================================
|
|
235
255
|
// Support-Tier Registry Schema
|
|
236
256
|
// =============================================================================
|
|
@@ -465,6 +485,9 @@ export {
|
|
|
465
485
|
DEFAULT_LOG_HISTORY_CONFIG,
|
|
466
486
|
DEFAULT_LOG_LEVEL_CONFIG,
|
|
467
487
|
DEFAULT_TRACE_CONFIG,
|
|
488
|
+
DEFAULT_LOADER_DEBUG_CONFIG,
|
|
489
|
+
DEFAULT_MATMUL_DEBUG_CONFIG,
|
|
490
|
+
DEFAULT_KERNEL_TRACE_CONFIG,
|
|
468
491
|
DEFAULT_PIPELINE_DEBUG_CONFIG,
|
|
469
492
|
DEFAULT_PROFILER_CONFIG,
|
|
470
493
|
DEFAULT_PERF_GUARDS_CONFIG,
|
|
@@ -551,6 +574,7 @@ export {
|
|
|
551
574
|
type BridgeConfigSchema,
|
|
552
575
|
|
|
553
576
|
// Defaults
|
|
577
|
+
DEFAULT_BRIDGE_TIMEOUT_CONFIG,
|
|
554
578
|
DEFAULT_BRIDGE_CONFIG,
|
|
555
579
|
} from './bridge.schema.js';
|
|
556
580
|
|
|
@@ -856,9 +880,36 @@ export {
|
|
|
856
880
|
|
|
857
881
|
// Defaults
|
|
858
882
|
DEFAULT_LARGE_WEIGHT_CONFIG,
|
|
883
|
+
DEFAULT_CHAT_TEMPLATE_CONFIG,
|
|
859
884
|
DEFAULT_RUNTIME_CONFIG,
|
|
860
885
|
DEFAULT_DOPPLER_CONFIG,
|
|
861
886
|
|
|
862
887
|
// Factory
|
|
863
888
|
createDopplerConfig,
|
|
864
889
|
} from './doppler.schema.js';
|
|
890
|
+
|
|
891
|
+
// =============================================================================
|
|
892
|
+
// Adapter Schema
|
|
893
|
+
// =============================================================================
|
|
894
|
+
export {
|
|
895
|
+
VALID_LORA_TARGET_MODULES,
|
|
896
|
+
DEFAULT_ADAPTER_VALIDATION_CONFIG,
|
|
897
|
+
DEFAULT_ADAPTER_STACK_CONFIG,
|
|
898
|
+
DEFAULT_ADAPTER_REGISTRY_CONFIG,
|
|
899
|
+
DEFAULT_ADAPTER_CONFIG,
|
|
900
|
+
} from './adapter.schema.js';
|
|
901
|
+
|
|
902
|
+
// =============================================================================
|
|
903
|
+
// Tuner Limits
|
|
904
|
+
// =============================================================================
|
|
905
|
+
export { DEFAULT_TUNER_LIMITS } from './tuner.schema.js';
|
|
906
|
+
|
|
907
|
+
// =============================================================================
|
|
908
|
+
// Self-Speculation Schema
|
|
909
|
+
// =============================================================================
|
|
910
|
+
export {
|
|
911
|
+
SPECULATION_MODES,
|
|
912
|
+
SPECULATION_VERIFY_MODES,
|
|
913
|
+
DEFAULT_SELF_SPECULATION_CONFIG,
|
|
914
|
+
validateSelfSpeculationConfig,
|
|
915
|
+
} from './speculation-self.schema.js';
|
|
@@ -62,6 +62,19 @@ export {
|
|
|
62
62
|
validateBrowserSuiteMetrics,
|
|
63
63
|
} from './browser-suite-metrics.schema.js';
|
|
64
64
|
|
|
65
|
+
// =============================================================================
|
|
66
|
+
// Program Bundle Schema
|
|
67
|
+
// =============================================================================
|
|
68
|
+
export {
|
|
69
|
+
PROGRAM_BUNDLE_SCHEMA_VERSION,
|
|
70
|
+
PROGRAM_BUNDLE_SCHEMA_ID,
|
|
71
|
+
PROGRAM_BUNDLE_HOST_SCHEMA_ID,
|
|
72
|
+
PROGRAM_BUNDLE_HOST_JS_SUBSET,
|
|
73
|
+
PROGRAM_BUNDLE_CAPTURE_PROFILE_SCHEMA_ID,
|
|
74
|
+
PROGRAM_BUNDLE_REFERENCE_TRANSCRIPT_SCHEMA_ID,
|
|
75
|
+
validateProgramBundle,
|
|
76
|
+
} from './program-bundle.schema.js';
|
|
77
|
+
|
|
65
78
|
// =============================================================================
|
|
66
79
|
// Support-Tier Registry Schema
|
|
67
80
|
// =============================================================================
|
|
@@ -327,6 +340,7 @@ export {
|
|
|
327
340
|
DEFAULT_TRAINING_PRECISION_CONFIG,
|
|
328
341
|
DEFAULT_TRAINING_ATTENTION_CONFIG,
|
|
329
342
|
DEFAULT_TRAINING_LOSS_SCALING_CONFIG,
|
|
343
|
+
DEFAULT_TRAINING_TELEMETRY_CONFIG,
|
|
330
344
|
DEFAULT_TRAINING_SETTINGS,
|
|
331
345
|
} from './training.schema.js';
|
|
332
346
|
|
|
@@ -282,9 +282,9 @@ export interface InferenceConfigSchema {
|
|
|
282
282
|
/** Chat template for instruct models */
|
|
283
283
|
chatTemplate?: ChatTemplateSchema;
|
|
284
284
|
/**
|
|
285
|
-
*
|
|
286
|
-
*
|
|
287
|
-
*
|
|
285
|
+
* Inline kernel path for explicit kernel dispatch ordering.
|
|
286
|
+
* String registry IDs were removed; execution-v1 manifests normally derive
|
|
287
|
+
* this object during compile. Use null for no explicit override.
|
|
288
288
|
*/
|
|
289
289
|
kernelPath?: KernelPathRef;
|
|
290
290
|
}
|
|
@@ -172,34 +172,13 @@ export interface KernelPathSchema {
|
|
|
172
172
|
}
|
|
173
173
|
|
|
174
174
|
/**
|
|
175
|
-
*
|
|
176
|
-
*
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
| 'gemma2-q4k-dequant-f16a' // Gemma 2 Q4K -> F16 dequant, F16 activations
|
|
181
|
-
| 'gemma2-q4k-dequant-f32a-nosubgroups' // Gemma 2 Q4K -> F32 dequant path with no subgroup requirement
|
|
182
|
-
| 'gemma2-q4k-dequant-f32a' // Legacy alias for gemma2-q4k-dequant-f32a-nosubgroups
|
|
183
|
-
| 'gemma2-f16-f16a' // Gemma 2 F16 weights, F16 activations
|
|
184
|
-
| 'gemma2-f16-f32a' // Gemma 2 F16 weights, F32 activations
|
|
185
|
-
| 'gemma3-f16-fused-f16a-online' // Gemma 3 F16 fused FFN online path
|
|
186
|
-
| 'gemma3-f16-fused-f32a-online' // Gemma 3 F16 fused FFN online path with F32 activations
|
|
187
|
-
| 'gemma3-f16-fused-f32a-online-streamingprefill' // Gemma 3 F16 fused FFN online path with streaming prefill attention
|
|
188
|
-
| 'gemma3-q4k-dequant-f16a-online' // Gemma 3 Q4K dequant online path (F16 activations)
|
|
189
|
-
| 'gemma3-q4k-dequant-f32a-online' // Gemma 3 Q4K dequant online path with F32 activations
|
|
190
|
-
| 'gemma3-q4k-dequant-f32w-f32a-online' // Gemma 3 Q4K path with F32 projection weights and F32 activations
|
|
191
|
-
| 'gemma3-q4k-dequant-f32a-nosubgroups' // Gemma 3 Q4K dequant path with no subgroup requirement
|
|
192
|
-
| 'gemma3-q4k-dequant-f32a' // Legacy alias for gemma3-q4k-dequant-f32a-nosubgroups
|
|
193
|
-
| 'lfm2-q4k-dequant-f32a-online' // LFM2 Q4K path with F32 activations and fast prefill
|
|
194
|
-
| 'embeddinggemma-f16-f32a' // EmbeddingGemma F16 weights, F32 activations
|
|
195
|
-
| 'embeddinggemma-f32-f32a' // EmbeddingGemma F32 weights, F32 activations
|
|
196
|
-
| 'embeddinggemma-q4k-dequant-f32a'; // EmbeddingGemma Q4K dequant, F32 activations
|
|
197
|
-
|
|
198
|
-
/**
|
|
199
|
-
* Kernel path reference - registered ID (string) or inline path schema.
|
|
200
|
-
* Accepts any string for custom IDs, not just built-in IDs.
|
|
175
|
+
* Kernel path reference.
|
|
176
|
+
*
|
|
177
|
+
* Registry string IDs were removed with the execution-v1 migration. Runtime
|
|
178
|
+
* and manifest surfaces now accept only inline kernel path objects generated
|
|
179
|
+
* from a pinned execution graph, or null for no explicit override.
|
|
201
180
|
*/
|
|
202
|
-
export type KernelPathRef =
|
|
181
|
+
export type KernelPathRef = KernelPathSchema | null;
|
|
203
182
|
|
|
204
183
|
/** Default entry point */
|
|
205
184
|
export declare const DEFAULT_ENTRY: string;
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { cloneJsonValue as cloneThresholdTree } from '../../utils/clone-json.js';
|
|
2
|
+
|
|
1
3
|
// =============================================================================
|
|
2
4
|
// Matmul Thresholds
|
|
3
5
|
// =============================================================================
|
|
@@ -91,7 +93,7 @@ export const DEFAULT_ATTENTION_THRESHOLDS = {
|
|
|
91
93
|
// Fused Matmul Thresholds
|
|
92
94
|
// =============================================================================
|
|
93
95
|
|
|
94
|
-
|
|
96
|
+
const DEFAULT_FUSED_MATMUL_THRESHOLDS = {
|
|
95
97
|
maxMediumN: 4096,
|
|
96
98
|
maxMediumK: 8192,
|
|
97
99
|
colsPerWg: 4,
|
|
@@ -155,12 +157,6 @@ export const DEFAULT_KERNEL_THRESHOLDS = {
|
|
|
155
157
|
tuner: DEFAULT_TUNER_LIMITS,
|
|
156
158
|
};
|
|
157
159
|
|
|
158
|
-
function cloneThresholdTree(value) {
|
|
159
|
-
if (typeof structuredClone === 'function') {
|
|
160
|
-
return structuredClone(value);
|
|
161
|
-
}
|
|
162
|
-
return JSON.parse(JSON.stringify(value));
|
|
163
|
-
}
|
|
164
160
|
|
|
165
161
|
// =============================================================================
|
|
166
162
|
// Runtime Access
|
|
@@ -37,12 +37,3 @@ export const DEFAULT_KVCACHE_CONFIG = {
|
|
|
37
37
|
|
|
38
38
|
export const PAGED_LAYOUT_SEQ_LEN_THRESHOLD = 8192;
|
|
39
39
|
|
|
40
|
-
const VALID_KV_DTYPES = new Set(['f16', 'f32']);
|
|
41
|
-
|
|
42
|
-
export function validateKvCacheDtype(dtype) {
|
|
43
|
-
if (dtype != null && !VALID_KV_DTYPES.has(dtype)) {
|
|
44
|
-
throw new Error(
|
|
45
|
-
`DopplerConfigError: kvcache.kvDtype must be "f16" or "f32"; got ${JSON.stringify(dtype)}.`
|
|
46
|
-
);
|
|
47
|
-
}
|
|
48
|
-
}
|
|
@@ -41,6 +41,15 @@ export interface ShardCacheConfigSchema {
|
|
|
41
41
|
|
|
42
42
|
/** Max concurrent shard loads (0 = unlimited) */
|
|
43
43
|
maxConcurrentLoads: number;
|
|
44
|
+
|
|
45
|
+
/** Coalesced range-fetch block size in bytes (0 = disabled) */
|
|
46
|
+
rangeCacheBlockBytes: number;
|
|
47
|
+
|
|
48
|
+
/** Maximum resident bytes for coalesced range-fetch blocks (0 = disabled) */
|
|
49
|
+
rangeCacheMaxBytes: number;
|
|
50
|
+
|
|
51
|
+
/** Minimum requested range size that may use coalesced range-fetch blocks */
|
|
52
|
+
rangeCacheMinBytes: number;
|
|
44
53
|
}
|
|
45
54
|
|
|
46
55
|
/** Default shard cache configuration */
|
|
@@ -108,6 +117,9 @@ export interface PrefetchConfigSchema {
|
|
|
108
117
|
|
|
109
118
|
/** Max shards to prefetch per layer step (0 = no limit) */
|
|
110
119
|
maxShards: number;
|
|
120
|
+
|
|
121
|
+
/** Allow whole-shard prefetch when a custom range loader is present */
|
|
122
|
+
allowRangeLoaderPrefetch: boolean;
|
|
111
123
|
}
|
|
112
124
|
|
|
113
125
|
/** Default prefetch configuration */
|
|
@@ -21,6 +21,9 @@ export const DEFAULT_SHARD_CACHE_CONFIG = {
|
|
|
21
21
|
moeMaxEntries: 16,
|
|
22
22
|
verifyHashes: true,
|
|
23
23
|
maxConcurrentLoads: 0,
|
|
24
|
+
rangeCacheBlockBytes: 0,
|
|
25
|
+
rangeCacheMaxBytes: 0,
|
|
26
|
+
rangeCacheMinBytes: 0,
|
|
24
27
|
};
|
|
25
28
|
|
|
26
29
|
// =============================================================================
|
|
@@ -51,6 +54,7 @@ export const DEFAULT_PREFETCH_CONFIG = {
|
|
|
51
54
|
enabled: true,
|
|
52
55
|
layersAhead: 1,
|
|
53
56
|
maxShards: 8,
|
|
57
|
+
allowRangeLoaderPrefetch: false,
|
|
54
58
|
};
|
|
55
59
|
|
|
56
60
|
// =============================================================================
|