doppler-gpu 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +19 -15
- package/package.json +77 -29
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +64 -6
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +524 -350
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +12 -2
- package/src/config/schema/doppler.schema.js +108 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +68 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +76 -0
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +107 -0
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +3 -21
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +6 -5
- package/src/gpu/kernels/index.js +3 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +12 -0
- package/src/inference/browser-harness-text-helpers.js +223 -14
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +264 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +22 -27
- package/src/inference/pipelines/text/embed.js +0 -39
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +4 -5
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +8 -3
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +223 -547
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +5 -2
- package/src/inference/pipelines/text/init.js +183 -724
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.js +2 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +120 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +223 -27
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +13 -3
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +11 -3
- package/src/storage/artifact-storage-context.js +231 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +21 -24
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +2 -0
- package/src/tooling/source-runtime-bundle.js +3 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
|
-
import { parseModelConfig } from './config.js';
|
|
4
3
|
import { getDevice, getDeviceLimits, getKernelCapabilities } from '../../../gpu/device.js';
|
|
5
|
-
import { acquireBuffer, isBufferActive, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
6
4
|
import {
|
|
7
5
|
KVCache,
|
|
8
6
|
SlidingWindowKVCache,
|
|
@@ -20,14 +18,13 @@ import { Tokenizer } from '../../tokenizer.js';
|
|
|
20
18
|
import { MoERouter } from '../../moe-router.js';
|
|
21
19
|
import { SpeculativeDecoder } from '../../speculative.js';
|
|
22
20
|
import { getDopplerLoader } from '../../../loader/doppler-loader.js';
|
|
23
|
-
import { log,
|
|
21
|
+
import { log, trace as debugTrace } from '../../../debug/index.js';
|
|
24
22
|
import { getRuntimeConfig } from '../../../config/runtime.js';
|
|
25
23
|
import { PAGED_LAYOUT_SEQ_LEN_THRESHOLD } from '../../../config/schema/index.js';
|
|
26
24
|
import {
|
|
27
|
-
getKernelPathMatmulVariant,
|
|
28
25
|
kernelPathRequiresF32MatmulWeights,
|
|
29
26
|
} from '../../../config/kernel-path-loader.js';
|
|
30
|
-
import {
|
|
27
|
+
import { resolveCapabilityTransforms } from '../../../config/transforms/capability-transform-resolver.js';
|
|
31
28
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
32
29
|
import { resolvePerLayerInputsSession } from './generator-helpers.js';
|
|
33
30
|
import {
|
|
@@ -35,6 +32,34 @@ import {
|
|
|
35
32
|
createNodeFileArtifactStorageContext,
|
|
36
33
|
} from '../../../storage/artifact-storage-context.js';
|
|
37
34
|
|
|
35
|
+
// Extracted standalone modules
|
|
36
|
+
import { initRoPEFrequencies, isGPURoPEBuffers } from './init-rope.js';
|
|
37
|
+
import {
|
|
38
|
+
applyChatTemplate,
|
|
39
|
+
applyGemmaChatTemplate,
|
|
40
|
+
applyGemma4ChatTemplate,
|
|
41
|
+
applyLlama3ChatTemplate,
|
|
42
|
+
applyGptOssChatTemplate,
|
|
43
|
+
applyQwenChatTemplate,
|
|
44
|
+
isStopToken,
|
|
45
|
+
} from './init-chat-templates.js';
|
|
46
|
+
import { fuseQKVWeights } from './init-qkv-fusion.js';
|
|
47
|
+
import { rewriteWeightLoadError } from './load-errors.js';
|
|
48
|
+
import { toArrayBuffer } from '../../../utils/array-buffer.js';
|
|
49
|
+
|
|
50
|
+
// Re-exports for backwards compatibility
|
|
51
|
+
export { initRoPEFrequencies, isGPURoPEBuffers } from './init-rope.js';
|
|
52
|
+
export {
|
|
53
|
+
applyChatTemplate,
|
|
54
|
+
applyGemmaChatTemplate,
|
|
55
|
+
applyGemma4ChatTemplate,
|
|
56
|
+
applyLlama3ChatTemplate,
|
|
57
|
+
applyGptOssChatTemplate,
|
|
58
|
+
applyQwenChatTemplate,
|
|
59
|
+
isStopToken,
|
|
60
|
+
} from './init-chat-templates.js';
|
|
61
|
+
export { fuseQKVWeights } from './init-qkv-fusion.js';
|
|
62
|
+
|
|
38
63
|
function resolveErrorMessage(error) {
|
|
39
64
|
if (error && typeof error === 'object' && typeof error.message === 'string') {
|
|
40
65
|
return error.message;
|
|
@@ -87,16 +112,6 @@ function assertSupportedTurboQuantMode(mode, label) {
|
|
|
87
112
|
);
|
|
88
113
|
}
|
|
89
114
|
|
|
90
|
-
function toArrayBuffer(value, label) {
|
|
91
|
-
if (value instanceof ArrayBuffer) {
|
|
92
|
-
return value;
|
|
93
|
-
}
|
|
94
|
-
if (value instanceof Uint8Array) {
|
|
95
|
-
return value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength);
|
|
96
|
-
}
|
|
97
|
-
throw new Error(`${label} must return ArrayBuffer or Uint8Array.`);
|
|
98
|
-
}
|
|
99
|
-
|
|
100
115
|
function toUint8Array(value, label) {
|
|
101
116
|
if (value instanceof Uint8Array) {
|
|
102
117
|
return value;
|
|
@@ -107,6 +122,25 @@ function toUint8Array(value, label) {
|
|
|
107
122
|
throw new Error(`${label} must return ArrayBuffer or Uint8Array.`);
|
|
108
123
|
}
|
|
109
124
|
|
|
125
|
+
function createStorageContextLoadError(error, operation, index, context = {}) {
|
|
126
|
+
const message = error?.message || String(error);
|
|
127
|
+
const wrapped = new Error(
|
|
128
|
+
`storageContext.${operation} failed for shard ${index}: ${message}`,
|
|
129
|
+
error instanceof Error ? { cause: error } : undefined
|
|
130
|
+
);
|
|
131
|
+
wrapped.name = error?.name || 'Error';
|
|
132
|
+
if (error?.code !== undefined) {
|
|
133
|
+
wrapped.code = error.code;
|
|
134
|
+
}
|
|
135
|
+
wrapped.details = {
|
|
136
|
+
...(error?.details && typeof error.details === 'object' ? error.details : {}),
|
|
137
|
+
storageOperation: operation,
|
|
138
|
+
shardIndex: index,
|
|
139
|
+
...context,
|
|
140
|
+
};
|
|
141
|
+
return wrapped;
|
|
142
|
+
}
|
|
143
|
+
|
|
110
144
|
function isRDRRManifest(manifest) {
|
|
111
145
|
return manifest !== null && typeof manifest === 'object' && Array.isArray( (manifest).shards);
|
|
112
146
|
}
|
|
@@ -137,6 +171,23 @@ const Q4K_PROJECTION_OPS = new Set([
|
|
|
137
171
|
'linear_out_proj',
|
|
138
172
|
]);
|
|
139
173
|
|
|
174
|
+
const Q4K_MATMUL_ROLE_OPS = new Map([
|
|
175
|
+
['q_proj', 'q_proj'],
|
|
176
|
+
['k_proj', 'k_proj'],
|
|
177
|
+
['v_proj', 'v_proj'],
|
|
178
|
+
['o_proj', 'o_proj'],
|
|
179
|
+
['gate_proj', 'ffn_gate'],
|
|
180
|
+
['up_proj', 'ffn_up'],
|
|
181
|
+
['down_proj', 'ffn_down'],
|
|
182
|
+
['qkv_proj', 'qkv_proj'],
|
|
183
|
+
['linear_qkv_proj', 'linear_qkv_proj'],
|
|
184
|
+
['linear_z_proj', 'linear_z_proj'],
|
|
185
|
+
['linear_a_proj', 'linear_a_proj'],
|
|
186
|
+
['linear_b_proj', 'linear_b_proj'],
|
|
187
|
+
['linear_out_proj', 'linear_out_proj'],
|
|
188
|
+
['lm_head', 'lm_head'],
|
|
189
|
+
]);
|
|
190
|
+
|
|
140
191
|
function summarizeQ4KProjectionKernelKinds(kernelPath) {
|
|
141
192
|
const summary = {
|
|
142
193
|
denseProjectionKernels: [],
|
|
@@ -148,14 +199,15 @@ function summarizeQ4KProjectionKernelKinds(kernelPath) {
|
|
|
148
199
|
|
|
149
200
|
const appendPhase = (phase, steps) => {
|
|
150
201
|
for (const step of steps ?? []) {
|
|
151
|
-
|
|
202
|
+
const op = step?.op;
|
|
203
|
+
if (!Q4K_PROJECTION_OPS.has(op)) {
|
|
152
204
|
continue;
|
|
153
205
|
}
|
|
154
206
|
const kernel = String(step?.kernel ?? '');
|
|
155
207
|
if (!kernel) {
|
|
156
208
|
continue;
|
|
157
209
|
}
|
|
158
|
-
const descriptor = `${phase}.${
|
|
210
|
+
const descriptor = `${phase}.${op}:${kernel}#${String(step?.entry ?? 'main')}`;
|
|
159
211
|
if (kernel.startsWith('fused_matmul_q4')) {
|
|
160
212
|
summary.fusedProjectionKernels.push(descriptor);
|
|
161
213
|
} else if (kernel.startsWith('matmul_')) {
|
|
@@ -169,6 +221,31 @@ function summarizeQ4KProjectionKernelKinds(kernelPath) {
|
|
|
169
221
|
return summary;
|
|
170
222
|
}
|
|
171
223
|
|
|
224
|
+
function collectQ4KFusedRoles(kernelPath) {
|
|
225
|
+
const roles = new Set();
|
|
226
|
+
if (!kernelPath || typeof kernelPath !== 'object') {
|
|
227
|
+
return [];
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
const appendSteps = (steps) => {
|
|
231
|
+
for (const step of steps ?? []) {
|
|
232
|
+
const role = Q4K_MATMUL_ROLE_OPS.get(step?.op);
|
|
233
|
+
if (!role) {
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
const kernel = String(step?.kernel ?? '');
|
|
237
|
+
if (kernel.startsWith('fused_matmul_q4')) {
|
|
238
|
+
roles.add(role);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
};
|
|
242
|
+
|
|
243
|
+
appendSteps(kernelPath.decode?.steps);
|
|
244
|
+
appendSteps(kernelPath.prefill?.steps);
|
|
245
|
+
appendSteps(kernelPath.postLayer);
|
|
246
|
+
return [...roles].sort();
|
|
247
|
+
}
|
|
248
|
+
|
|
172
249
|
function resolveQ4KProjectionMaterializationMode(
|
|
173
250
|
manifest,
|
|
174
251
|
kernelPath,
|
|
@@ -187,6 +264,25 @@ function resolveQ4KProjectionMaterializationMode(
|
|
|
187
264
|
return mode;
|
|
188
265
|
}
|
|
189
266
|
|
|
267
|
+
function isRetainQ4KMaterializationDisabledByCapability(manifest, caps) {
|
|
268
|
+
if (!caps) return false;
|
|
269
|
+
const adapterInfo = caps.adapterInfo ?? {};
|
|
270
|
+
const platform = {
|
|
271
|
+
id: adapterInfo.device ?? 'unknown',
|
|
272
|
+
vendor: adapterInfo.vendor ?? 'unknown',
|
|
273
|
+
architecture: adapterInfo.architecture ?? 'unknown',
|
|
274
|
+
};
|
|
275
|
+
const runtimeSession = getRuntimeConfig().inference?.session ?? {};
|
|
276
|
+
const graphContext = {
|
|
277
|
+
modelId: manifest?.modelId ?? 'unknown',
|
|
278
|
+
activationDtype: runtimeSession.compute?.defaults?.activationDtype ?? null,
|
|
279
|
+
kvDtype: runtimeSession.kvcache?.kvDtype ?? null,
|
|
280
|
+
retainQ4KMaterialization: true,
|
|
281
|
+
};
|
|
282
|
+
const resolved = resolveCapabilityTransforms(caps, platform, graphContext);
|
|
283
|
+
return resolved.names.includes('disableRetainQ4KMaterialization');
|
|
284
|
+
}
|
|
285
|
+
|
|
190
286
|
export function resolveQ4KConfig(
|
|
191
287
|
manifest,
|
|
192
288
|
kernelPath,
|
|
@@ -198,6 +294,7 @@ export function resolveQ4KConfig(
|
|
|
198
294
|
// Layout in quantizationInfo: 'row' (fused) or 'col' (dequant)
|
|
199
295
|
const q4kLayout = manifest?.quantizationInfo?.layout ?? null;
|
|
200
296
|
const isQ4KModel = manifest?.quantization === 'Q4_K_M';
|
|
297
|
+
const q4kFusedRoles = isQ4KModel ? collectQ4KFusedRoles(kernelPath) : [];
|
|
201
298
|
if (isQ4KModel && q4kLayout == null) {
|
|
202
299
|
throw new Error(
|
|
203
300
|
`Manifest "${manifest?.modelId ?? 'unknown'}" is missing quantizationInfo.layout for Q4_K_M. Re-convert the model.`
|
|
@@ -209,9 +306,33 @@ export function resolveQ4KConfig(
|
|
|
209
306
|
`Allowed values: ${[...Q4K_LAYOUT_ALLOWLIST].join(', ')}.`
|
|
210
307
|
);
|
|
211
308
|
}
|
|
212
|
-
|
|
309
|
+
let q4kMaterializationMode = isQ4KModel
|
|
213
310
|
? resolveQ4KProjectionMaterializationMode(manifest, kernelPath, kernelPathSource)
|
|
214
311
|
: 'dense';
|
|
312
|
+
// Runtime override: retainQ4KMaterialization forces "mixed" mode so the
|
|
313
|
+
// loader keeps the Q4_K packed buffer alongside the dequantized dense
|
|
314
|
+
// buffer. Unlocks `hasQ4KMaterialization=true` in the FFN fusion rule when
|
|
315
|
+
// the execution graph doesn't declare a fused projection kernel.
|
|
316
|
+
if (isQ4KModel && hasSubgroups && q4kMaterializationMode === 'dense') {
|
|
317
|
+
// Runtime-over-manifest precedence per config-style-guide §Category Rules:
|
|
318
|
+
// getRuntimeConfig() returns the merged session (manifest base + runtime
|
|
319
|
+
// overrides via merge.js); consumers should not re-implement precedence.
|
|
320
|
+
const runtimeRetain = getRuntimeConfig().inference?.session?.retainQ4KMaterialization === true;
|
|
321
|
+
if (runtimeRetain) {
|
|
322
|
+
if (isRetainQ4KMaterializationDisabledByCapability(manifest, caps)) {
|
|
323
|
+
debugTrace.loader(
|
|
324
|
+
'Q4K materialization retain request disabled by capability rule: ' +
|
|
325
|
+
`model=${manifest?.modelId ?? 'unknown'}, adapter=${caps.adapterInfo?.vendor ?? 'unknown'}/` +
|
|
326
|
+
`${caps.adapterInfo?.architecture ?? 'unknown'}, mode=${q4kMaterializationMode}`
|
|
327
|
+
);
|
|
328
|
+
} else {
|
|
329
|
+
q4kMaterializationMode = 'mixed';
|
|
330
|
+
debugTrace.loader(
|
|
331
|
+
`Q4K materialization overridden by runtime flag retainQ4KMaterialization=true: mode=mixed`
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
}
|
|
215
336
|
if (isQ4KModel) {
|
|
216
337
|
debugTrace.loader(
|
|
217
338
|
`Q4K projection materialization: model=${manifest?.modelId ?? 'unknown'}, ` +
|
|
@@ -241,414 +362,10 @@ export function resolveQ4KConfig(
|
|
|
241
362
|
q4kLayout,
|
|
242
363
|
keepF32Weights: resolvedKeepF32Weights,
|
|
243
364
|
q4kMaterializationMode,
|
|
365
|
+
q4kFusedRoles,
|
|
244
366
|
};
|
|
245
367
|
}
|
|
246
368
|
|
|
247
|
-
// ============================================================================
|
|
248
|
-
// RoPE Initialization
|
|
249
|
-
// ============================================================================
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
function computeRoPEFreqsForTheta(
|
|
253
|
-
theta,
|
|
254
|
-
rotaryDim,
|
|
255
|
-
frequencyBaseDim,
|
|
256
|
-
maxSeqLen,
|
|
257
|
-
ropeScale,
|
|
258
|
-
ropeScalingType,
|
|
259
|
-
ropeScaling
|
|
260
|
-
) {
|
|
261
|
-
const halfDim = rotaryDim / 2;
|
|
262
|
-
|
|
263
|
-
// Compute base frequencies: theta_i = 1 / (base^(2i/d))
|
|
264
|
-
const freqs = new Float32Array(halfDim);
|
|
265
|
-
for (let i = 0; i < halfDim; i++) {
|
|
266
|
-
freqs[i] = 1.0 / Math.pow(theta, (2 * i) / frequencyBaseDim);
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
// Compute per-dimension scaling factors
|
|
270
|
-
const scales = new Float32Array(halfDim);
|
|
271
|
-
const isYarn = ropeScalingType === 'yarn';
|
|
272
|
-
if (isYarn) {
|
|
273
|
-
// YARN scaling - validate ALL required params (fail fast on incomplete manifest)
|
|
274
|
-
if (ropeScaling?.factor == null ||
|
|
275
|
-
ropeScaling?.beta_fast == null || ropeScaling?.beta_slow == null ||
|
|
276
|
-
ropeScaling?.original_max_position_embeddings == null) {
|
|
277
|
-
throw new Error(
|
|
278
|
-
`RoPE scaling type is 'yarn' but YARN params missing. ` +
|
|
279
|
-
`Manifest must provide factor, beta_fast, beta_slow, and original_max_position_embeddings. ` +
|
|
280
|
-
`Got: factor=${ropeScaling?.factor}, beta_fast=${ropeScaling?.beta_fast}, beta_slow=${ropeScaling?.beta_slow}, ` +
|
|
281
|
-
`original_max_position_embeddings=${ropeScaling?.original_max_position_embeddings}`
|
|
282
|
-
);
|
|
283
|
-
}
|
|
284
|
-
// Extract validated YARN params (no hidden defaults - all guaranteed non-null)
|
|
285
|
-
const yarnFactor = ropeScaling.factor;
|
|
286
|
-
const yarnBetaFast = ropeScaling.beta_fast;
|
|
287
|
-
const yarnBetaSlow = ropeScaling.beta_slow;
|
|
288
|
-
const originalMaxPos = ropeScaling.original_max_position_embeddings;
|
|
289
|
-
|
|
290
|
-
// YARN: wavelength-based interpolation
|
|
291
|
-
for (let i = 0; i < halfDim; i++) {
|
|
292
|
-
const wavelength = (2 * Math.PI) / freqs[i];
|
|
293
|
-
const lowThresh = originalMaxPos / yarnBetaSlow;
|
|
294
|
-
const highThresh = originalMaxPos / yarnBetaFast;
|
|
295
|
-
|
|
296
|
-
if (wavelength < highThresh) {
|
|
297
|
-
scales[i] = 1.0;
|
|
298
|
-
} else if (wavelength > lowThresh) {
|
|
299
|
-
scales[i] = yarnFactor;
|
|
300
|
-
} else {
|
|
301
|
-
const t = (wavelength - highThresh) / (lowThresh - highThresh);
|
|
302
|
-
scales[i] = 1.0 + (yarnFactor - 1.0) * t;
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
} else {
|
|
306
|
-
// Linear scaling: uniform across all dimensions
|
|
307
|
-
if (ropeScalingType != null && ropeScalingType !== 'linear') {
|
|
308
|
-
log.warn(
|
|
309
|
-
'Pipeline',
|
|
310
|
-
`Unrecognized RoPE scaling type "${ropeScalingType}"; falling back to linear scaling. ` +
|
|
311
|
-
'Known types: "linear", "yarn".'
|
|
312
|
-
);
|
|
313
|
-
}
|
|
314
|
-
for (let i = 0; i < halfDim; i++) {
|
|
315
|
-
scales[i] = ropeScale;
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
// Compute cos/sin for each position
|
|
320
|
-
const cosValues = new Float32Array(maxSeqLen * halfDim);
|
|
321
|
-
const sinValues = new Float32Array(maxSeqLen * halfDim);
|
|
322
|
-
|
|
323
|
-
for (let pos = 0; pos < maxSeqLen; pos++) {
|
|
324
|
-
for (let i = 0; i < halfDim; i++) {
|
|
325
|
-
const scaledPos = pos / scales[i];
|
|
326
|
-
const angle = scaledPos * freqs[i];
|
|
327
|
-
cosValues[pos * halfDim + i] = Math.cos(angle);
|
|
328
|
-
sinValues[pos * halfDim + i] = Math.sin(angle);
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
return { cos: cosValues, sin: sinValues };
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
function isSameRoPEScalingConfig(
|
|
336
|
-
leftType,
|
|
337
|
-
leftScale,
|
|
338
|
-
leftScaling,
|
|
339
|
-
rightType,
|
|
340
|
-
rightScale,
|
|
341
|
-
rightScaling
|
|
342
|
-
) {
|
|
343
|
-
if (leftType !== rightType) return false;
|
|
344
|
-
if (leftScale !== rightScale) return false;
|
|
345
|
-
if (leftType !== 'yarn') return true;
|
|
346
|
-
return (leftScaling?.beta_fast ?? null) === (rightScaling?.beta_fast ?? null)
|
|
347
|
-
&& (leftScaling?.beta_slow ?? null) === (rightScaling?.beta_slow ?? null)
|
|
348
|
-
&& (leftScaling?.original_max_position_embeddings ?? null)
|
|
349
|
-
=== (rightScaling?.original_max_position_embeddings ?? null);
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
const GPU_ROPE_BUFFER_CACHE = new WeakMap();
|
|
353
|
-
const CPU_ROPE_BUFFER_CACHE = new Map();
|
|
354
|
-
|
|
355
|
-
function isLiveCachedRopeBuffer(buffer) {
|
|
356
|
-
return buffer == null || isBufferActive(buffer);
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
function hasLiveCachedGpuRopeBuffers(buffers) {
|
|
360
|
-
return !!buffers
|
|
361
|
-
&& isLiveCachedRopeBuffer(buffers.cos)
|
|
362
|
-
&& isLiveCachedRopeBuffer(buffers.sin)
|
|
363
|
-
&& isLiveCachedRopeBuffer(buffers.localCos)
|
|
364
|
-
&& isLiveCachedRopeBuffer(buffers.localSin);
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
function buildRoPECacheKey(config) {
|
|
368
|
-
return JSON.stringify({
|
|
369
|
-
headDim: config.headDim,
|
|
370
|
-
localHeadDim: config.localHeadDim ?? null,
|
|
371
|
-
rotaryDim: config.rotaryDim ?? null,
|
|
372
|
-
ropeLocalRotaryDim: config.ropeLocalRotaryDim ?? null,
|
|
373
|
-
ropeFrequencyBaseDim: config.ropeFrequencyBaseDim ?? null,
|
|
374
|
-
ropeLocalFrequencyBaseDim: config.ropeLocalFrequencyBaseDim ?? null,
|
|
375
|
-
maxSeqLen: config.maxSeqLen,
|
|
376
|
-
ropeTheta: config.ropeTheta,
|
|
377
|
-
ropeLocalTheta: config.ropeLocalTheta ?? null,
|
|
378
|
-
mropeInterleaved: config.mropeInterleaved === true,
|
|
379
|
-
mropeSection: Array.isArray(config.mropeSection) ? [...config.mropeSection] : null,
|
|
380
|
-
partialRotaryFactor: config.partialRotaryFactor ?? null,
|
|
381
|
-
ropeLocalPartialRotaryFactor: config.ropeLocalPartialRotaryFactor ?? null,
|
|
382
|
-
ropeScale: config.ropeScale,
|
|
383
|
-
ropeLocalScale: config.ropeLocalScale ?? null,
|
|
384
|
-
ropeScalingType: config.ropeScalingType ?? null,
|
|
385
|
-
ropeLocalScalingType: config.ropeLocalScalingType ?? null,
|
|
386
|
-
ropeScaling: config.ropeScaling ?? null,
|
|
387
|
-
ropeLocalScaling: config.ropeLocalScaling ?? null,
|
|
388
|
-
});
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
function resolveRotaryDim(headDim, rotaryDim, partialRotaryFactor) {
|
|
392
|
-
if (rotaryDim != null) {
|
|
393
|
-
if (!Number.isFinite(rotaryDim) || rotaryDim <= 0 || (rotaryDim % 2) !== 0) {
|
|
394
|
-
throw new Error(`RoPE rotary dim must be a positive even integer; got "${rotaryDim}".`);
|
|
395
|
-
}
|
|
396
|
-
if (rotaryDim > headDim) {
|
|
397
|
-
throw new Error(`RoPE rotary dim ${rotaryDim} cannot exceed headDim ${headDim}.`);
|
|
398
|
-
}
|
|
399
|
-
return rotaryDim;
|
|
400
|
-
}
|
|
401
|
-
if (partialRotaryFactor == null) {
|
|
402
|
-
return headDim;
|
|
403
|
-
}
|
|
404
|
-
if (!Number.isFinite(partialRotaryFactor) || partialRotaryFactor <= 0 || partialRotaryFactor > 1) {
|
|
405
|
-
throw new Error(
|
|
406
|
-
`RoPE partialRotaryFactor must be a number in (0, 1]; got "${partialRotaryFactor}".`
|
|
407
|
-
);
|
|
408
|
-
}
|
|
409
|
-
const resolved = Math.trunc(headDim * partialRotaryFactor);
|
|
410
|
-
if (resolved <= 0 || (resolved % 2) !== 0) {
|
|
411
|
-
throw new Error(
|
|
412
|
-
`RoPE partialRotaryFactor=${partialRotaryFactor} with headDim=${headDim} resolves ` +
|
|
413
|
-
`to rotaryDim=${resolved}, but rotaryDim must be a positive even integer.`
|
|
414
|
-
);
|
|
415
|
-
}
|
|
416
|
-
return resolved;
|
|
417
|
-
}
|
|
418
|
-
|
|
419
|
-
function resolveFrequencyBaseDim(rotaryDim, frequencyBaseDim, label) {
|
|
420
|
-
if (frequencyBaseDim == null) {
|
|
421
|
-
return rotaryDim;
|
|
422
|
-
}
|
|
423
|
-
if (!Number.isFinite(frequencyBaseDim) || frequencyBaseDim <= 0 || (Math.trunc(frequencyBaseDim) % 2) !== 0) {
|
|
424
|
-
throw new Error(`${label} must be a positive even integer; got "${frequencyBaseDim}".`);
|
|
425
|
-
}
|
|
426
|
-
const resolved = Math.trunc(frequencyBaseDim);
|
|
427
|
-
if (resolved < rotaryDim) {
|
|
428
|
-
throw new Error(`${label} ${resolved} cannot be smaller than rotaryDim ${rotaryDim}.`);
|
|
429
|
-
}
|
|
430
|
-
return resolved;
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
export async function initRoPEFrequencies(config, useGPU) {
|
|
435
|
-
const cacheKey = buildRoPECacheKey(config);
|
|
436
|
-
const {
|
|
437
|
-
headDim,
|
|
438
|
-
localHeadDim,
|
|
439
|
-
rotaryDim,
|
|
440
|
-
ropeLocalRotaryDim,
|
|
441
|
-
ropeFrequencyBaseDim,
|
|
442
|
-
ropeLocalFrequencyBaseDim,
|
|
443
|
-
maxSeqLen,
|
|
444
|
-
ropeTheta,
|
|
445
|
-
ropeLocalTheta,
|
|
446
|
-
mropeInterleaved,
|
|
447
|
-
mropeSection,
|
|
448
|
-
partialRotaryFactor,
|
|
449
|
-
ropeLocalPartialRotaryFactor,
|
|
450
|
-
ropeScale,
|
|
451
|
-
ropeLocalScale,
|
|
452
|
-
ropeScalingType,
|
|
453
|
-
ropeLocalScalingType,
|
|
454
|
-
ropeScaling,
|
|
455
|
-
ropeLocalScaling,
|
|
456
|
-
} = config;
|
|
457
|
-
if (!Number.isFinite(ropeScale) || ropeScale <= 0) {
|
|
458
|
-
throw new Error(`RoPE scale must be a positive number; got "${ropeScale}".`);
|
|
459
|
-
}
|
|
460
|
-
const resolvedLocalScale = ropeLocalScale;
|
|
461
|
-
if (resolvedLocalScale != null && (!Number.isFinite(resolvedLocalScale) || resolvedLocalScale <= 0)) {
|
|
462
|
-
throw new Error(`Local RoPE scale must be a positive number; got "${resolvedLocalScale}".`);
|
|
463
|
-
}
|
|
464
|
-
const resolvedLocalTheta = ropeLocalTheta ?? ropeTheta;
|
|
465
|
-
const resolvedLocalScalingType = (
|
|
466
|
-
ropeLocalScalingType === undefined
|
|
467
|
-
? ropeScalingType
|
|
468
|
-
: ropeLocalScalingType
|
|
469
|
-
);
|
|
470
|
-
const resolvedLocalScaling = (
|
|
471
|
-
ropeLocalScalingType === undefined
|
|
472
|
-
? ropeScaling
|
|
473
|
-
: ropeLocalScaling
|
|
474
|
-
);
|
|
475
|
-
const resolvedLocalHeadDim = localHeadDim ?? headDim;
|
|
476
|
-
const resolvedRotaryDim = resolveRotaryDim(headDim, rotaryDim, partialRotaryFactor);
|
|
477
|
-
const resolvedLocalRotaryDim = resolveRotaryDim(
|
|
478
|
-
resolvedLocalHeadDim,
|
|
479
|
-
ropeLocalRotaryDim,
|
|
480
|
-
ropeLocalPartialRotaryFactor
|
|
481
|
-
);
|
|
482
|
-
const resolvedFrequencyBaseDim = resolveFrequencyBaseDim(
|
|
483
|
-
resolvedRotaryDim,
|
|
484
|
-
ropeFrequencyBaseDim,
|
|
485
|
-
'RoPE frequency base dim'
|
|
486
|
-
);
|
|
487
|
-
const resolvedLocalFrequencyBaseDim = resolveFrequencyBaseDim(
|
|
488
|
-
resolvedLocalRotaryDim,
|
|
489
|
-
ropeLocalFrequencyBaseDim,
|
|
490
|
-
'Local RoPE frequency base dim'
|
|
491
|
-
);
|
|
492
|
-
const halfDim = resolvedRotaryDim / 2;
|
|
493
|
-
if (mropeInterleaved === true && Array.isArray(mropeSection)) {
|
|
494
|
-
const expandedDim = mropeSection.reduce((sum, entry) => sum + entry, 0) * 2;
|
|
495
|
-
if (expandedDim !== resolvedRotaryDim) {
|
|
496
|
-
throw new Error(
|
|
497
|
-
`RoPE mropeSection expands to ${expandedDim} dims, but rotaryDim is ${resolvedRotaryDim}.`
|
|
498
|
-
);
|
|
499
|
-
}
|
|
500
|
-
}
|
|
501
|
-
|
|
502
|
-
const isYarn = ropeScalingType === 'yarn';
|
|
503
|
-
const isLocalYarn = resolvedLocalScalingType === 'yarn';
|
|
504
|
-
|
|
505
|
-
// Compute global (full_attention) frequencies
|
|
506
|
-
const globalFreqs = computeRoPEFreqsForTheta(
|
|
507
|
-
ropeTheta,
|
|
508
|
-
resolvedRotaryDim,
|
|
509
|
-
resolvedFrequencyBaseDim,
|
|
510
|
-
maxSeqLen,
|
|
511
|
-
ropeScale,
|
|
512
|
-
ropeScalingType,
|
|
513
|
-
ropeScaling
|
|
514
|
-
);
|
|
515
|
-
|
|
516
|
-
// Compute local (sliding_attention) frequencies if different from global.
|
|
517
|
-
// Models with dual RoPE use different theta for local vs global attention layers.
|
|
518
|
-
|
|
519
|
-
let localFreqs = null;
|
|
520
|
-
const hasDistinctLocalTheta = resolvedLocalTheta !== ropeTheta;
|
|
521
|
-
const hasDistinctLocalDim = resolvedLocalRotaryDim !== resolvedRotaryDim;
|
|
522
|
-
const hasDistinctLocalScaling = !isSameRoPEScalingConfig(
|
|
523
|
-
ropeScalingType,
|
|
524
|
-
ropeScale,
|
|
525
|
-
ropeScaling,
|
|
526
|
-
resolvedLocalScalingType,
|
|
527
|
-
resolvedLocalScale,
|
|
528
|
-
resolvedLocalScaling
|
|
529
|
-
);
|
|
530
|
-
if (hasDistinctLocalTheta || hasDistinctLocalScaling || hasDistinctLocalDim) {
|
|
531
|
-
localFreqs = computeRoPEFreqsForTheta(
|
|
532
|
-
resolvedLocalTheta,
|
|
533
|
-
resolvedLocalRotaryDim,
|
|
534
|
-
resolvedLocalFrequencyBaseDim,
|
|
535
|
-
maxSeqLen,
|
|
536
|
-
resolvedLocalScale,
|
|
537
|
-
resolvedLocalScalingType,
|
|
538
|
-
resolvedLocalScaling
|
|
539
|
-
);
|
|
540
|
-
log.debug(
|
|
541
|
-
'Pipeline',
|
|
542
|
-
`Dual RoPE: local theta=${resolvedLocalTheta}, global theta=${ropeTheta}, ` +
|
|
543
|
-
`localRotaryDim=${resolvedLocalRotaryDim}, globalRotaryDim=${resolvedRotaryDim}, ` +
|
|
544
|
-
`localFrequencyBaseDim=${resolvedLocalFrequencyBaseDim}, globalFrequencyBaseDim=${resolvedFrequencyBaseDim}, ` +
|
|
545
|
-
`localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}, ` +
|
|
546
|
-
`globalScaling=${ropeScalingType ?? 'none'}:${ropeScale}`
|
|
547
|
-
);
|
|
548
|
-
}
|
|
549
|
-
|
|
550
|
-
if (isYarn) {
|
|
551
|
-
// Log YARN params (already validated in computeRoPEFreqs)
|
|
552
|
-
log.debug('Pipeline', `YARN RoPE: factor=${ropeScaling?.factor}, beta_fast=${ropeScaling?.beta_fast}, beta_slow=${ropeScaling?.beta_slow}`);
|
|
553
|
-
}
|
|
554
|
-
if (isLocalYarn && hasDistinctLocalScaling) {
|
|
555
|
-
log.debug(
|
|
556
|
-
'Pipeline',
|
|
557
|
-
`Local YARN RoPE: factor=${resolvedLocalScaling?.factor}, ` +
|
|
558
|
-
`beta_fast=${resolvedLocalScaling?.beta_fast}, beta_slow=${resolvedLocalScaling?.beta_slow}`
|
|
559
|
-
);
|
|
560
|
-
}
|
|
561
|
-
|
|
562
|
-
// Upload to GPU if available
|
|
563
|
-
const device = getDevice();
|
|
564
|
-
if (device && useGPU) {
|
|
565
|
-
let perDeviceCache = GPU_ROPE_BUFFER_CACHE.get(device);
|
|
566
|
-
if (!perDeviceCache) {
|
|
567
|
-
perDeviceCache = new Map();
|
|
568
|
-
GPU_ROPE_BUFFER_CACHE.set(device, perDeviceCache);
|
|
569
|
-
}
|
|
570
|
-
const cachedBuffers = perDeviceCache.get(cacheKey);
|
|
571
|
-
if (cachedBuffers) {
|
|
572
|
-
if (hasLiveCachedGpuRopeBuffers(cachedBuffers)) {
|
|
573
|
-
return cachedBuffers;
|
|
574
|
-
}
|
|
575
|
-
perDeviceCache.delete(cacheKey);
|
|
576
|
-
}
|
|
577
|
-
let cosBuffer = null;
|
|
578
|
-
let sinBuffer = null;
|
|
579
|
-
let localCosBuffer = null;
|
|
580
|
-
let localSinBuffer = null;
|
|
581
|
-
try {
|
|
582
|
-
cosBuffer = acquireBuffer(globalFreqs.cos.byteLength, undefined, 'rope_cos');
|
|
583
|
-
sinBuffer = acquireBuffer(globalFreqs.sin.byteLength, undefined, 'rope_sin');
|
|
584
|
-
device.queue.writeBuffer(cosBuffer, 0, globalFreqs.cos.buffer, globalFreqs.cos.byteOffset, globalFreqs.cos.byteLength);
|
|
585
|
-
device.queue.writeBuffer(sinBuffer, 0, globalFreqs.sin.buffer, globalFreqs.sin.byteOffset, globalFreqs.sin.byteLength);
|
|
586
|
-
|
|
587
|
-
if (localFreqs) {
|
|
588
|
-
localCosBuffer = acquireBuffer(localFreqs.cos.byteLength, undefined, 'rope_local_cos');
|
|
589
|
-
localSinBuffer = acquireBuffer(localFreqs.sin.byteLength, undefined, 'rope_local_sin');
|
|
590
|
-
device.queue.writeBuffer(localCosBuffer, 0, localFreqs.cos.buffer, localFreqs.cos.byteOffset, localFreqs.cos.byteLength);
|
|
591
|
-
device.queue.writeBuffer(localSinBuffer, 0, localFreqs.sin.buffer, localFreqs.sin.byteOffset, localFreqs.sin.byteLength);
|
|
592
|
-
}
|
|
593
|
-
} catch (error) {
|
|
594
|
-
for (const buffer of [cosBuffer, sinBuffer, localCosBuffer, localSinBuffer]) {
|
|
595
|
-
if (buffer) {
|
|
596
|
-
releaseBuffer(buffer);
|
|
597
|
-
}
|
|
598
|
-
}
|
|
599
|
-
throw error;
|
|
600
|
-
}
|
|
601
|
-
|
|
602
|
-
log.debug(
|
|
603
|
-
'Pipeline',
|
|
604
|
-
`RoPE frequencies initialized (GPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
|
|
605
|
-
`theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
|
|
606
|
-
`${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
|
|
607
|
-
`scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
|
|
608
|
-
`interleaved=${mropeInterleaved === true}`
|
|
609
|
-
);
|
|
610
|
-
|
|
611
|
-
const buffers = {
|
|
612
|
-
cos: cosBuffer,
|
|
613
|
-
sin: sinBuffer,
|
|
614
|
-
localCos: localCosBuffer,
|
|
615
|
-
localSin: localSinBuffer,
|
|
616
|
-
};
|
|
617
|
-
perDeviceCache.set(cacheKey, buffers);
|
|
618
|
-
return buffers;
|
|
619
|
-
}
|
|
620
|
-
|
|
621
|
-
const cachedCpuBuffers = CPU_ROPE_BUFFER_CACHE.get(cacheKey);
|
|
622
|
-
if (cachedCpuBuffers) {
|
|
623
|
-
return cachedCpuBuffers;
|
|
624
|
-
}
|
|
625
|
-
|
|
626
|
-
log.debug(
|
|
627
|
-
'Pipeline',
|
|
628
|
-
`RoPE frequencies initialized (CPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
|
|
629
|
-
`theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
|
|
630
|
-
`${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
|
|
631
|
-
`scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
|
|
632
|
-
`interleaved=${mropeInterleaved === true}`
|
|
633
|
-
);
|
|
634
|
-
|
|
635
|
-
const buffers = {
|
|
636
|
-
cos: globalFreqs.cos,
|
|
637
|
-
sin: globalFreqs.sin,
|
|
638
|
-
localCos: localFreqs?.cos,
|
|
639
|
-
localSin: localFreqs?.sin,
|
|
640
|
-
};
|
|
641
|
-
CPU_ROPE_BUFFER_CACHE.set(cacheKey, buffers);
|
|
642
|
-
return buffers;
|
|
643
|
-
}
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
export function isGPURoPEBuffers(buffers) {
|
|
647
|
-
if (typeof GPUBuffer === 'undefined') return false;
|
|
648
|
-
return !!buffers?.cos && isGpuBufferInstance(buffers.cos);
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
|
|
652
369
|
function normalizeLayerType(layerType) {
|
|
653
370
|
return typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
|
|
654
371
|
}
|
|
@@ -973,6 +690,7 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
|
|
|
973
690
|
kvCache = new MixedGeometryKVCache({
|
|
974
691
|
...cacheConfig,
|
|
975
692
|
numHeads: modelConfig.numKVHeads,
|
|
693
|
+
globalNumHeads: modelConfig.numGlobalKVHeads ?? modelConfig.numKVHeads,
|
|
976
694
|
globalHeadDim: modelConfig.globalHeadDim ?? null,
|
|
977
695
|
slidingWindow,
|
|
978
696
|
layerTypes: modelConfig.layerTypes,
|
|
@@ -1086,6 +804,9 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
1086
804
|
} = options;
|
|
1087
805
|
const runtimeStorageContext = options.storageContext
|
|
1088
806
|
?? createRemoteStorageContext(baseUrl, manifest);
|
|
807
|
+
if (typeof runtimeStorageContext?.preflight === 'function') {
|
|
808
|
+
await runtimeStorageContext.preflight();
|
|
809
|
+
}
|
|
1089
810
|
const verifyHashes = (
|
|
1090
811
|
typeof runtimeStorageContext?.verifyHashes === 'boolean'
|
|
1091
812
|
? runtimeStorageContext.verifyHashes
|
|
@@ -1121,6 +842,11 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
1121
842
|
} else {
|
|
1122
843
|
dopplerLoader.setTensorsJsonUrl(null);
|
|
1123
844
|
}
|
|
845
|
+
dopplerLoader.setTensorsJsonLoader(
|
|
846
|
+
typeof runtimeStorageContext?.loadTensorsJson === 'function'
|
|
847
|
+
? () => runtimeStorageContext.loadTensorsJson()
|
|
848
|
+
: null
|
|
849
|
+
);
|
|
1124
850
|
|
|
1125
851
|
// Configure custom shard loader if provided (Native Bridge or direct-source bundle)
|
|
1126
852
|
const hasLoadShard = typeof runtimeStorageContext?.loadShard === 'function';
|
|
@@ -1130,25 +856,37 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
1130
856
|
log.debug('Pipeline', 'Using custom shard loader (Native Bridge or external)');
|
|
1131
857
|
|
|
1132
858
|
const loadShard = async (index) => {
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
859
|
+
try {
|
|
860
|
+
if (hasLoadShard) {
|
|
861
|
+
const data = await runtimeStorageContext.loadShard(index);
|
|
862
|
+
return toUint8Array(data, 'storageContext.loadShard');
|
|
863
|
+
}
|
|
864
|
+
const rangeData = await runtimeStorageContext.loadShardRange(index, 0, null);
|
|
865
|
+
return toUint8Array(rangeData, 'storageContext.loadShardRange');
|
|
866
|
+
} catch (error) {
|
|
867
|
+
throw createStorageContextLoadError(error, hasLoadShard ? 'loadShard' : 'loadShardRange', index);
|
|
1136
868
|
}
|
|
1137
|
-
const rangeData = await runtimeStorageContext.loadShardRange(index, 0, null);
|
|
1138
|
-
return toUint8Array(rangeData, 'storageContext.loadShardRange');
|
|
1139
869
|
};
|
|
1140
870
|
|
|
1141
871
|
const loadShardRange = hasLoadShardRange
|
|
1142
872
|
? async (index, offset, length = null) => {
|
|
1143
|
-
|
|
1144
|
-
|
|
873
|
+
try {
|
|
874
|
+
const data = await runtimeStorageContext.loadShardRange(index, offset, length);
|
|
875
|
+
return toArrayBuffer(data, 'storageContext.loadShardRange');
|
|
876
|
+
} catch (error) {
|
|
877
|
+
throw createStorageContextLoadError(error, 'loadShardRange', index, { offset, length });
|
|
878
|
+
}
|
|
1145
879
|
}
|
|
1146
880
|
: null;
|
|
1147
881
|
|
|
1148
882
|
const streamShardRange = hasStreamShardRange
|
|
1149
883
|
? async function* (index, offset = 0, length = null, streamOptions = {}) {
|
|
1150
|
-
|
|
1151
|
-
|
|
884
|
+
try {
|
|
885
|
+
for await (const chunk of runtimeStorageContext.streamShardRange(index, offset, length, streamOptions)) {
|
|
886
|
+
yield toUint8Array(chunk, 'storageContext.streamShardRange');
|
|
887
|
+
}
|
|
888
|
+
} catch (error) {
|
|
889
|
+
throw createStorageContextLoadError(error, 'streamShardRange', index, { offset, length });
|
|
1152
890
|
}
|
|
1153
891
|
}
|
|
1154
892
|
: null;
|
|
@@ -1170,15 +908,19 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
1170
908
|
if (!modelId) {
|
|
1171
909
|
throw new Error('Manifest is missing modelId. Re-convert the model with modelId set.');
|
|
1172
910
|
}
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
911
|
+
try {
|
|
912
|
+
await dopplerLoader.load(modelId, {
|
|
913
|
+
verifyHashes,
|
|
914
|
+
onProgress: onProgress || ((info) => {
|
|
915
|
+
// Shard and layer progress are logged by loader with source info
|
|
916
|
+
if (info.stage !== 'layers' && info.stage !== 'shards') {
|
|
917
|
+
log.verbose('Loader', `${info.stage}: ${Math.round(info.progress * 100)}%`);
|
|
918
|
+
}
|
|
919
|
+
}),
|
|
920
|
+
});
|
|
921
|
+
} catch (error) {
|
|
922
|
+
throw rewriteWeightLoadError(error, { modelId });
|
|
923
|
+
}
|
|
1182
924
|
|
|
1183
925
|
// Map layer weights
|
|
1184
926
|
|
|
@@ -1217,93 +959,6 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
1217
959
|
};
|
|
1218
960
|
}
|
|
1219
961
|
|
|
1220
|
-
// ============================================================================
|
|
1221
|
-
// Chat Templates
|
|
1222
|
-
// ============================================================================
|
|
1223
|
-
|
|
1224
|
-
// Simple prompt templates for single-turn chat.
|
|
1225
|
-
// For multi-turn conversations, use formatChatMessages from chat-format.js.
|
|
1226
|
-
|
|
1227
|
-
function applyTurnBasedTemplate(prompt) {
|
|
1228
|
-
// Turn-based format: <start_of_turn>role\ncontent<end_of_turn>
|
|
1229
|
-
const userTurn = `<start_of_turn>user\n${prompt}<end_of_turn>\n`;
|
|
1230
|
-
const modelTurn = '<start_of_turn>model\n';
|
|
1231
|
-
return userTurn + modelTurn;
|
|
1232
|
-
}
|
|
1233
|
-
|
|
1234
|
-
function applyHeaderBasedTemplate(prompt) {
|
|
1235
|
-
// Header-based format: <|start_header_id|>role<|end_header_id|>\n\ncontent<|eot_id|>
|
|
1236
|
-
return `<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n${prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n`;
|
|
1237
|
-
}
|
|
1238
|
-
|
|
1239
|
-
function applyChannelBasedTemplate(prompt) {
|
|
1240
|
-
// Channel-based format: <|start|>role<|message|>content<|end|>
|
|
1241
|
-
return `<|start|>user<|message|>${prompt}<|end|><|start|>assistant<|channel|>final<|message|>`;
|
|
1242
|
-
}
|
|
1243
|
-
|
|
1244
|
-
function applyChatMLTemplate(prompt) {
|
|
1245
|
-
// ChatML format: <|im_start|>role\ncontent<|im_end|>
|
|
1246
|
-
return `<|im_start|>user\n${prompt}<|im_end|>\n<|im_start|>assistant\n`;
|
|
1247
|
-
}
|
|
1248
|
-
|
|
1249
|
-
function getQwenAssistantPrefix(options) {
|
|
1250
|
-
return options?.thinking === true
|
|
1251
|
-
? '<|im_start|>assistant\n<think>\n'
|
|
1252
|
-
: '<|im_start|>assistant\n<think>\n\n</think>\n\n';
|
|
1253
|
-
}
|
|
1254
|
-
|
|
1255
|
-
function applyQwenTemplate(prompt, options) {
|
|
1256
|
-
return `<|im_start|>user\n${prompt}<|im_end|>\n${getQwenAssistantPrefix(options)}`;
|
|
1257
|
-
}
|
|
1258
|
-
|
|
1259
|
-
function applyGemma4Template(prompt) {
|
|
1260
|
-
return `<bos><|turn>user\n${prompt}<turn|>\n<|turn>model\n`;
|
|
1261
|
-
}
|
|
1262
|
-
|
|
1263
|
-
function applyTranslateGemmaTemplate() {
|
|
1264
|
-
throw new Error(
|
|
1265
|
-
'TranslateGemma template requires structured messages. ' +
|
|
1266
|
-
'Use formatChatMessages(messages, "translategemma") instead of applyChatTemplate(prompt, ...).'
|
|
1267
|
-
);
|
|
1268
|
-
}
|
|
1269
|
-
|
|
1270
|
-
// Template type to formatter mapping.
|
|
1271
|
-
// Add new template types here rather than adding switch cases.
|
|
1272
|
-
const PROMPT_TEMPLATES = {
|
|
1273
|
-
'gemma': applyTurnBasedTemplate,
|
|
1274
|
-
'gemma4': applyGemma4Template,
|
|
1275
|
-
'llama3': applyHeaderBasedTemplate,
|
|
1276
|
-
'gpt-oss': applyChannelBasedTemplate,
|
|
1277
|
-
'chatml': applyChatMLTemplate,
|
|
1278
|
-
'qwen': applyQwenTemplate,
|
|
1279
|
-
'translategemma': applyTranslateGemmaTemplate,
|
|
1280
|
-
};
|
|
1281
|
-
|
|
1282
|
-
export function applyChatTemplate(prompt, templateType, options = undefined) {
|
|
1283
|
-
if (templateType == null) {
|
|
1284
|
-
return prompt;
|
|
1285
|
-
}
|
|
1286
|
-
const formatter = PROMPT_TEMPLATES[templateType];
|
|
1287
|
-
if (formatter) {
|
|
1288
|
-
return formatter(prompt, options);
|
|
1289
|
-
}
|
|
1290
|
-
throw new Error(`Unrecognized chat template type: ${templateType}`);
|
|
1291
|
-
}
|
|
1292
|
-
|
|
1293
|
-
// Exports preserved for existing external imports.
|
|
1294
|
-
export const applyGemmaChatTemplate = applyTurnBasedTemplate;
|
|
1295
|
-
export const applyGemma4ChatTemplate = applyGemma4Template;
|
|
1296
|
-
export const applyLlama3ChatTemplate = applyHeaderBasedTemplate;
|
|
1297
|
-
export const applyGptOssChatTemplate = applyChannelBasedTemplate;
|
|
1298
|
-
export const applyQwenChatTemplate = applyQwenTemplate;
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
export function isStopToken(token, stopTokenIds, eosTokenId) {
|
|
1302
|
-
if (stopTokenIds.includes(token)) return true;
|
|
1303
|
-
if (typeof eosTokenId === 'number' && token === eosTokenId) return true;
|
|
1304
|
-
return false;
|
|
1305
|
-
}
|
|
1306
|
-
|
|
1307
962
|
// ============================================================================
|
|
1308
963
|
// MoE Router Setup
|
|
1309
964
|
// ============================================================================
|
|
@@ -1361,202 +1016,6 @@ export function initSpeculativeDecoder(manifest, speculativeConfig) {
|
|
|
1361
1016
|
randomSeed: speculativeConfig.randomSeed,
|
|
1362
1017
|
});
|
|
1363
1018
|
}
|
|
1364
|
-
|
|
1365
|
-
// ============================================================================
|
|
1366
|
-
// QKV Fusion
|
|
1367
|
-
// ============================================================================
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
function kernelPathVariantRequiresQ4KWeights(variant) {
|
|
1371
|
-
if (typeof variant !== 'string' || variant.length === 0) {
|
|
1372
|
-
return false;
|
|
1373
|
-
}
|
|
1374
|
-
let config;
|
|
1375
|
-
try {
|
|
1376
|
-
config = getKernelConfig('matmul', variant);
|
|
1377
|
-
} catch {
|
|
1378
|
-
return false;
|
|
1379
|
-
}
|
|
1380
|
-
const shaderFile = String(config?.shaderFile ?? config?.wgsl ?? '');
|
|
1381
|
-
return shaderFile.startsWith('fused_matmul_q4');
|
|
1382
|
-
}
|
|
1383
|
-
|
|
1384
|
-
function shouldSkipQKVFusionForLayer(layerIdx, kernelPath) {
|
|
1385
|
-
if (!kernelPath) {
|
|
1386
|
-
return false;
|
|
1387
|
-
}
|
|
1388
|
-
const prefillVariant = getKernelPathMatmulVariant('qkv_proj', 'prefill', layerIdx, kernelPath);
|
|
1389
|
-
if (kernelPathVariantRequiresQ4KWeights(prefillVariant)) {
|
|
1390
|
-
return true;
|
|
1391
|
-
}
|
|
1392
|
-
const decodeVariant = getKernelPathMatmulVariant('qkv_proj', 'decode', layerIdx, kernelPath);
|
|
1393
|
-
return kernelPathVariantRequiresQ4KWeights(decodeVariant);
|
|
1394
|
-
}
|
|
1395
|
-
|
|
1396
|
-
export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
|
|
1397
|
-
const device = getDevice();
|
|
1398
|
-
if (!device) {
|
|
1399
|
-
log.debug('QKV Fusion', 'No GPU device, skipping fusion');
|
|
1400
|
-
return;
|
|
1401
|
-
}
|
|
1402
|
-
if (
|
|
1403
|
-
Number.isFinite(modelConfig?.globalHeadDim)
|
|
1404
|
-
&& modelConfig.globalHeadDim !== modelConfig.headDim
|
|
1405
|
-
) {
|
|
1406
|
-
log.debug(
|
|
1407
|
-
'QKV Fusion',
|
|
1408
|
-
'Skipping QKV fusion for mixed-head-dim model; per-layer attention geometry must stay explicit.'
|
|
1409
|
-
);
|
|
1410
|
-
return;
|
|
1411
|
-
}
|
|
1412
|
-
|
|
1413
|
-
const { numLayers, numHeads, numKVHeads, headDim, hiddenSize } = modelConfig;
|
|
1414
|
-
const qSize = numHeads * headDim;
|
|
1415
|
-
const kSize = numKVHeads * headDim;
|
|
1416
|
-
const vSize = numKVHeads * headDim;
|
|
1417
|
-
const qkvSize = qSize + kSize + vSize;
|
|
1418
|
-
|
|
1419
|
-
const resolveWeight = (value) => {
|
|
1420
|
-
if (isWeightBuffer(value)) {
|
|
1421
|
-
return {
|
|
1422
|
-
buffer: value.buffer,
|
|
1423
|
-
dtype: value.dtype ?? null,
|
|
1424
|
-
layout: value.layout ?? null,
|
|
1425
|
-
shape: Array.isArray(value.shape) ? value.shape : null,
|
|
1426
|
-
};
|
|
1427
|
-
}
|
|
1428
|
-
if (isGpuBufferInstance(value)) {
|
|
1429
|
-
return {
|
|
1430
|
-
buffer: value,
|
|
1431
|
-
dtype: getWeightDtype(value),
|
|
1432
|
-
layout: null,
|
|
1433
|
-
shape: null,
|
|
1434
|
-
};
|
|
1435
|
-
}
|
|
1436
|
-
return null;
|
|
1437
|
-
};
|
|
1438
|
-
|
|
1439
|
-
log.debug('QKV Fusion', `Fusing Q/K/V weights for ${numLayers} layers (${qSize}+${kSize}+${vSize}=${qkvSize})`);
|
|
1440
|
-
|
|
1441
|
-
const resolveBytesPerElement = (weight, expectedElements) => {
|
|
1442
|
-
const dtype = typeof weight?.dtype === 'string'
|
|
1443
|
-
? weight.dtype.toLowerCase()
|
|
1444
|
-
: null;
|
|
1445
|
-
if (dtype === 'f16' || dtype === 'bf16') return 2;
|
|
1446
|
-
if (dtype === 'f32') return 4;
|
|
1447
|
-
const minF16Bytes = expectedElements * 2;
|
|
1448
|
-
const minF32Bytes = expectedElements * 4;
|
|
1449
|
-
if (weight?.buffer?.size >= minF32Bytes) return 4;
|
|
1450
|
-
if (weight?.buffer?.size >= minF16Bytes) return 2;
|
|
1451
|
-
return 0;
|
|
1452
|
-
};
|
|
1453
|
-
|
|
1454
|
-
let fusedCount = 0;
|
|
1455
|
-
for (let l = 0; l < numLayers; l++) {
|
|
1456
|
-
const weights = layerWeights.get(`layer_${l}`);
|
|
1457
|
-
if (!weights) continue;
|
|
1458
|
-
|
|
1459
|
-
// Skip if already fused or if weights are not GPUBuffers
|
|
1460
|
-
if (weights.qkvProj) continue;
|
|
1461
|
-
if (shouldSkipQKVFusionForLayer(l, kernelPath)) {
|
|
1462
|
-
log.debug(
|
|
1463
|
-
'QKV Fusion',
|
|
1464
|
-
`Layer ${l}: skipped because active kernel path requires Q4K weights for qkv_proj`
|
|
1465
|
-
);
|
|
1466
|
-
continue;
|
|
1467
|
-
}
|
|
1468
|
-
const qProj = resolveWeight(weights.qProj);
|
|
1469
|
-
const kProj = resolveWeight(weights.kProj);
|
|
1470
|
-
const vProj = resolveWeight(weights.vProj);
|
|
1471
|
-
if (!qProj || !kProj || !vProj) {
|
|
1472
|
-
continue;
|
|
1473
|
-
}
|
|
1474
|
-
|
|
1475
|
-
const qExpectedElements = qSize * hiddenSize;
|
|
1476
|
-
const kExpectedElements = kSize * hiddenSize;
|
|
1477
|
-
const vExpectedElements = vSize * hiddenSize;
|
|
1478
|
-
const bytesPerElement = resolveBytesPerElement(qProj, qExpectedElements);
|
|
1479
|
-
const kBytesPerElement = resolveBytesPerElement(kProj, kExpectedElements);
|
|
1480
|
-
const vBytesPerElement = resolveBytesPerElement(vProj, vExpectedElements);
|
|
1481
|
-
|
|
1482
|
-
// Pool allocation can round GPUBuffer.size up, so infer logical dtype first and
|
|
1483
|
-
// only use buffer size as a minimum-size inference.
|
|
1484
|
-
if ((bytesPerElement !== 2 && bytesPerElement !== 4)
|
|
1485
|
-
|| kBytesPerElement !== bytesPerElement
|
|
1486
|
-
|| vBytesPerElement !== bytesPerElement) {
|
|
1487
|
-
log.debug(
|
|
1488
|
-
'QKV Fusion',
|
|
1489
|
-
`Layer ${l}: inconsistent projection dtypes (q=${bytesPerElement}, k=${kBytesPerElement}, v=${vBytesPerElement}), skipping`
|
|
1490
|
-
);
|
|
1491
|
-
continue;
|
|
1492
|
-
}
|
|
1493
|
-
|
|
1494
|
-
const normalizedDtype = typeof qProj.dtype === 'string'
|
|
1495
|
-
? qProj.dtype.toLowerCase()
|
|
1496
|
-
: null;
|
|
1497
|
-
const dtype = normalizedDtype === 'bf16'
|
|
1498
|
-
? 'f16'
|
|
1499
|
-
: (
|
|
1500
|
-
normalizedDtype === 'f16' || normalizedDtype === 'f32'
|
|
1501
|
-
? normalizedDtype
|
|
1502
|
-
: selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement })
|
|
1503
|
-
);
|
|
1504
|
-
const layout = qProj.layout ?? kProj.layout ?? vProj.layout ?? 'row';
|
|
1505
|
-
let fusedShape = [qkvSize, hiddenSize];
|
|
1506
|
-
if (Array.isArray(qProj.shape) && qProj.shape.length === 2) {
|
|
1507
|
-
if (qProj.shape[0] === qSize && qProj.shape[1] === hiddenSize) {
|
|
1508
|
-
fusedShape = [qkvSize, hiddenSize];
|
|
1509
|
-
} else if (qProj.shape[1] === qSize && qProj.shape[0] === hiddenSize) {
|
|
1510
|
-
fusedShape = [hiddenSize, qkvSize];
|
|
1511
|
-
}
|
|
1512
|
-
}
|
|
1513
|
-
|
|
1514
|
-
// Create fused QKV buffer: [qkvSize, hiddenSize] row-major
|
|
1515
|
-
// Each row is concatenated: [q_row, k_row, v_row]
|
|
1516
|
-
const qkvBuffer = device.createBuffer({
|
|
1517
|
-
label: `layer_${l}_qkv_proj`,
|
|
1518
|
-
size: qkvSize * hiddenSize * bytesPerElement,
|
|
1519
|
-
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
|
|
1520
|
-
});
|
|
1521
|
-
|
|
1522
|
-
// Copy Q, K, V weights into fused buffer
|
|
1523
|
-
// Q: [qSize, hiddenSize] -> offset 0
|
|
1524
|
-
// K: [kSize, hiddenSize] -> offset qSize * hiddenSize * bytesPerElement
|
|
1525
|
-
// V: [vSize, hiddenSize] -> offset (qSize + kSize) * hiddenSize * bytesPerElement
|
|
1526
|
-
const encoder = device.createCommandEncoder({ label: 'qkv_fusion' });
|
|
1527
|
-
encoder.copyBufferToBuffer(
|
|
1528
|
-
qProj.buffer, 0,
|
|
1529
|
-
qkvBuffer, 0,
|
|
1530
|
-
qSize * hiddenSize * bytesPerElement
|
|
1531
|
-
);
|
|
1532
|
-
encoder.copyBufferToBuffer(
|
|
1533
|
-
kProj.buffer, 0,
|
|
1534
|
-
qkvBuffer, qSize * hiddenSize * bytesPerElement,
|
|
1535
|
-
kSize * hiddenSize * bytesPerElement
|
|
1536
|
-
);
|
|
1537
|
-
encoder.copyBufferToBuffer(
|
|
1538
|
-
vProj.buffer, 0,
|
|
1539
|
-
qkvBuffer, (qSize + kSize) * hiddenSize * bytesPerElement,
|
|
1540
|
-
vSize * hiddenSize * bytesPerElement
|
|
1541
|
-
);
|
|
1542
|
-
device.queue.submit([encoder.finish()]);
|
|
1543
|
-
|
|
1544
|
-
// Store fused buffer, sizes, and dtype
|
|
1545
|
-
weights.qkvProj = createWeightBuffer(
|
|
1546
|
-
qkvBuffer,
|
|
1547
|
-
dtype,
|
|
1548
|
-
layout,
|
|
1549
|
-
fusedShape,
|
|
1550
|
-
`layer_${l}_qkv_proj`
|
|
1551
|
-
);
|
|
1552
|
-
weights.qkvSizes = [qSize, kSize, vSize];
|
|
1553
|
-
weights.qkvDtype = dtype;
|
|
1554
|
-
fusedCount++;
|
|
1555
|
-
}
|
|
1556
|
-
|
|
1557
|
-
log.debug('QKV Fusion', `Fused ${fusedCount}/${numLayers} layers`);
|
|
1558
|
-
}
|
|
1559
|
-
|
|
1560
1019
|
// ============================================================================
|
|
1561
1020
|
// Emulation Setup
|
|
1562
1021
|
// ============================================================================
|