doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -5,6 +5,7 @@ import type { CommandRecorder } from '../../../gpu/command-recorder.js';
|
|
|
5
5
|
import type { LinearNormMode } from '../../../config/schema/index.js';
|
|
6
6
|
import type { ProbeConfigSchema } from '../../../config/schema/index.js';
|
|
7
7
|
import type { ExecutionV1PoliciesSchema } from '../../../config/schema/execution-v1.schema.js';
|
|
8
|
+
import type { KernelPathSchema } from '../../../config/schema/kernel-path.schema.js';
|
|
8
9
|
|
|
9
10
|
export interface LinearLayerRuntimeState {
|
|
10
11
|
layerIdx: number;
|
|
@@ -61,7 +62,7 @@ export interface RunLinearAttentionLayerOptions {
|
|
|
61
62
|
};
|
|
62
63
|
currentSeqLen: number;
|
|
63
64
|
activationDtype?: 'f16' | 'f32';
|
|
64
|
-
kernelPath?:
|
|
65
|
+
kernelPath?: KernelPathSchema | null;
|
|
65
66
|
executionPolicies?: ExecutionV1PoliciesSchema | null;
|
|
66
67
|
linearRuntime?: LinearAttentionRuntime | null;
|
|
67
68
|
getWeightBuffer: (
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export interface WeightLoadErrorDetails {
|
|
2
|
+
[key: string]: unknown;
|
|
3
|
+
}
|
|
4
|
+
|
|
5
|
+
export type AnnotatableError = Error & {
|
|
6
|
+
details?: Record<string, unknown>;
|
|
7
|
+
code?: string;
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
export declare function annotateWeightLoadError<T>(
|
|
11
|
+
error: T,
|
|
12
|
+
details?: WeightLoadErrorDetails
|
|
13
|
+
): T;
|
|
14
|
+
|
|
15
|
+
export declare function rewriteWeightLoadError(
|
|
16
|
+
error: unknown,
|
|
17
|
+
context?: {
|
|
18
|
+
modelId?: string | null;
|
|
19
|
+
deviceLossInfo?: Record<string, unknown> | null;
|
|
20
|
+
}
|
|
21
|
+
): unknown;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { getLastDeviceLossInfo } from '../../../gpu/device.js';
|
|
2
|
+
import { createDopplerError, ERROR_CODES } from '../../../errors/doppler-error.js';
|
|
3
|
+
|
|
4
|
+
const DEVICE_LIFECYCLE_PATTERNS = [
|
|
5
|
+
/device not initialized/i,
|
|
6
|
+
/gpu device not initialized/i,
|
|
7
|
+
/no gpu device available/i,
|
|
8
|
+
];
|
|
9
|
+
|
|
10
|
+
function matchesDeviceLifecycleFailure(message) {
|
|
11
|
+
return DEVICE_LIFECYCLE_PATTERNS.some((pattern) => pattern.test(message));
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function annotateWeightLoadError(error, details = {}) {
|
|
15
|
+
if (!error || typeof error !== 'object') {
|
|
16
|
+
return error;
|
|
17
|
+
}
|
|
18
|
+
const existingDetails = error.details && typeof error.details === 'object'
|
|
19
|
+
? error.details
|
|
20
|
+
: {};
|
|
21
|
+
error.details = {
|
|
22
|
+
...existingDetails,
|
|
23
|
+
weightLoadFailure: {
|
|
24
|
+
...(existingDetails.weightLoadFailure && typeof existingDetails.weightLoadFailure === 'object'
|
|
25
|
+
? existingDetails.weightLoadFailure
|
|
26
|
+
: {}),
|
|
27
|
+
...details,
|
|
28
|
+
},
|
|
29
|
+
};
|
|
30
|
+
return error;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function rewriteWeightLoadError(error, context = {}) {
|
|
34
|
+
const message = error?.message || String(error);
|
|
35
|
+
if (!matchesDeviceLifecycleFailure(message)) {
|
|
36
|
+
return error;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const modelId = typeof context.modelId === 'string' && context.modelId.trim()
|
|
40
|
+
? context.modelId.trim()
|
|
41
|
+
: 'unknown';
|
|
42
|
+
const lastDeviceLoss = context.deviceLossInfo ?? getLastDeviceLossInfo();
|
|
43
|
+
const wrapped = createDopplerError(
|
|
44
|
+
ERROR_CODES.GPU_DEVICE_FAILED,
|
|
45
|
+
`Weight load lost access to the WebGPU device for model "${modelId}". ` +
|
|
46
|
+
'This is a device lifecycle failure during loadWeights, not proof of VRAM-capacity exhaustion. ' +
|
|
47
|
+
'Check device initialization/loss logs and the first failing buffer allocation or upload.'
|
|
48
|
+
);
|
|
49
|
+
wrapped.cause = error;
|
|
50
|
+
wrapped.details = {
|
|
51
|
+
...(error?.details && typeof error.details === 'object' ? error.details : {}),
|
|
52
|
+
loadPhase: 'loadWeights',
|
|
53
|
+
modelId,
|
|
54
|
+
lifecycleFailure: 'device_unavailable_during_weight_load',
|
|
55
|
+
originalMessage: message,
|
|
56
|
+
lastDeviceLoss: lastDeviceLoss ? { ...lastDeviceLoss } : null,
|
|
57
|
+
};
|
|
58
|
+
return wrapped;
|
|
59
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { f16ToF32 } from '../../../../loader/dtype-utils.js';
|
|
1
2
|
|
|
2
|
-
|
|
3
|
+
export { f16ToF32 };
|
|
3
4
|
|
|
4
5
|
// Caller should pass eps from model config; inline default for safety
|
|
5
6
|
export function rmsNormCPU(
|
|
@@ -25,25 +26,6 @@ export function rmsNormCPU(
|
|
|
25
26
|
}
|
|
26
27
|
|
|
27
28
|
|
|
28
|
-
export function f16ToF32(h) {
|
|
29
|
-
const sign = (h >> 15) & 0x1;
|
|
30
|
-
const exp = (h >> 10) & 0x1f;
|
|
31
|
-
const mant = h & 0x3ff;
|
|
32
|
-
|
|
33
|
-
if (exp === 0) {
|
|
34
|
-
if (mant === 0) return sign ? -0 : 0;
|
|
35
|
-
const f = mant / 1024 * Math.pow(2, -14);
|
|
36
|
-
return sign ? -f : f;
|
|
37
|
-
}
|
|
38
|
-
if (exp === 31) {
|
|
39
|
-
return mant ? NaN : (sign ? -Infinity : Infinity);
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
const f = (1 + mant / 1024) * Math.pow(2, exp - 15);
|
|
43
|
-
return sign ? -f : f;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
|
|
47
29
|
export function f16BufferToF32(data) {
|
|
48
30
|
const u16 = new Uint16Array(data);
|
|
49
31
|
const out = new Float32Array(u16.length);
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import type { Tensor } from '../../../../gpu/tensor.js';
|
|
11
|
-
import type { CpuWeightBuffer } from '../../../../gpu/weight-buffer.js';
|
|
11
|
+
import type { CpuWeightBuffer, CpuTensorRangeSource } from '../../../../gpu/weight-buffer.js';
|
|
12
12
|
import type { CommandRecorder } from '../../../../gpu/command-recorder.js';
|
|
13
13
|
import type { LargeWeightConfigSchema, ProbeConfigSchema, KernelPathSchema } from '../../../../config/schema/index.js';
|
|
14
14
|
import type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
|
|
@@ -32,13 +32,14 @@ export function resolveLmHeadChunkRows(
|
|
|
32
32
|
* Extract a chunk of the LM head weight matrix.
|
|
33
33
|
*/
|
|
34
34
|
export function extractLmHeadChunk(
|
|
35
|
-
data: Float32Array,
|
|
35
|
+
data: Float32Array | Uint16Array | CpuTensorRangeSource,
|
|
36
36
|
layout: 'row' | 'column',
|
|
37
37
|
hiddenSize: number,
|
|
38
38
|
vocabSize: number,
|
|
39
39
|
rowOffset: number,
|
|
40
|
-
rowCount: number
|
|
41
|
-
|
|
40
|
+
rowCount: number,
|
|
41
|
+
sourceDtype?: 'f16' | 'f32' | 'bf16'
|
|
42
|
+
): Promise<Float32Array>;
|
|
42
43
|
|
|
43
44
|
/**
|
|
44
45
|
* Write chunk logits to the full logits buffer.
|
|
@@ -21,6 +21,7 @@ import { runProbes } from '../probes.js';
|
|
|
21
21
|
import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
|
|
22
22
|
import { f16BufferToF32 } from './cpu.js';
|
|
23
23
|
import { readBufferWithCleanup } from './utils.js';
|
|
24
|
+
import { f16ToF32 } from '../../../../loader/dtype-utils.js';
|
|
24
25
|
|
|
25
26
|
function shouldForceStableF32Logits(config, inputDtype) {
|
|
26
27
|
if (inputDtype !== 'f16') {
|
|
@@ -140,6 +141,101 @@ function createStableF32LogitsKernelPath(kernelPath) {
|
|
|
140
141
|
};
|
|
141
142
|
}
|
|
142
143
|
|
|
144
|
+
const bf16ScratchU32 = new Uint32Array(1);
|
|
145
|
+
const bf16ScratchF32 = new Float32Array(bf16ScratchU32.buffer);
|
|
146
|
+
|
|
147
|
+
function bf16ToF32(value) {
|
|
148
|
+
bf16ScratchU32[0] = (value & 0xffff) << 16;
|
|
149
|
+
return bf16ScratchF32[0];
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function isRangeBackedCpuWeightSource(value) {
|
|
153
|
+
return (
|
|
154
|
+
typeof value === 'object'
|
|
155
|
+
&& value !== null
|
|
156
|
+
&& value.kind === 'tensor_range_source'
|
|
157
|
+
&& typeof value.loadRange === 'function'
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function normalizeRangeBytes(value, label) {
|
|
162
|
+
if (value instanceof Uint8Array) return value;
|
|
163
|
+
if (value instanceof ArrayBuffer) return new Uint8Array(value);
|
|
164
|
+
if (ArrayBuffer.isView(value)) {
|
|
165
|
+
return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
|
|
166
|
+
}
|
|
167
|
+
throw new Error(
|
|
168
|
+
`[Logits] ${label} returned unsupported byte payload type "${value?.constructor?.name ?? typeof value}".`
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function decodeChunkIntoOutput(bytes, sourceDtype, output, dstOffset, valueCount) {
|
|
173
|
+
if (sourceDtype === 'f16') {
|
|
174
|
+
const values = new Uint16Array(bytes.buffer, bytes.byteOffset, valueCount);
|
|
175
|
+
for (let index = 0; index < valueCount; index += 1) {
|
|
176
|
+
output[dstOffset + index] = f16ToF32(values[index]);
|
|
177
|
+
}
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
if (sourceDtype === 'bf16') {
|
|
181
|
+
const values = new Uint16Array(bytes.buffer, bytes.byteOffset, valueCount);
|
|
182
|
+
for (let index = 0; index < valueCount; index += 1) {
|
|
183
|
+
output[dstOffset + index] = bf16ToF32(values[index]);
|
|
184
|
+
}
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
if (((bytes.byteOffset % 4) === 0) && ((bytes.byteLength % 4) === 0)) {
|
|
188
|
+
output.set(new Float32Array(bytes.buffer, bytes.byteOffset, valueCount), dstOffset);
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
192
|
+
for (let index = 0; index < valueCount; index += 1) {
|
|
193
|
+
output[dstOffset + index] = view.getFloat32(index * 4, true);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function readTypedLmHeadChunk(data, layout, hiddenSize, vocabSize, rowOffset, rowCount, sourceDtype) {
|
|
198
|
+
if (data instanceof Float32Array) {
|
|
199
|
+
if (layout === 'row') {
|
|
200
|
+
const start = rowOffset * hiddenSize;
|
|
201
|
+
return data.subarray(start, start + rowCount * hiddenSize);
|
|
202
|
+
}
|
|
203
|
+
const chunk = new Float32Array(hiddenSize * rowCount);
|
|
204
|
+
for (let k = 0; k < hiddenSize; k++) {
|
|
205
|
+
const srcOffset = k * vocabSize + rowOffset;
|
|
206
|
+
const dstOffset = k * rowCount;
|
|
207
|
+
chunk.set(data.subarray(srcOffset, srcOffset + rowCount), dstOffset);
|
|
208
|
+
}
|
|
209
|
+
return chunk;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
if (!(data instanceof Uint16Array)) {
|
|
213
|
+
throw new Error(
|
|
214
|
+
`[Logits] Unsupported CPU LM head chunk source type "${data?.constructor?.name ?? typeof data}".`
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
const chunk = new Float32Array(hiddenSize * rowCount);
|
|
219
|
+
if (layout === 'row') {
|
|
220
|
+
const start = rowOffset * hiddenSize;
|
|
221
|
+
for (let index = 0; index < rowCount * hiddenSize; index += 1) {
|
|
222
|
+
const raw = data[start + index];
|
|
223
|
+
chunk[index] = sourceDtype === 'bf16' ? bf16ToF32(raw) : f16ToF32(raw);
|
|
224
|
+
}
|
|
225
|
+
return chunk;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
for (let k = 0; k < hiddenSize; k += 1) {
|
|
229
|
+
const srcOffset = k * vocabSize + rowOffset;
|
|
230
|
+
const dstOffset = k * rowCount;
|
|
231
|
+
for (let index = 0; index < rowCount; index += 1) {
|
|
232
|
+
const raw = data[srcOffset + index];
|
|
233
|
+
chunk[dstOffset + index] = sourceDtype === 'bf16' ? bf16ToF32(raw) : f16ToF32(raw);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
return chunk;
|
|
237
|
+
}
|
|
238
|
+
|
|
143
239
|
|
|
144
240
|
export function resolveCpuWeightDims(lmHead) {
|
|
145
241
|
if (lmHead.shape.length !== 2) {
|
|
@@ -184,24 +280,62 @@ export function resolveLmHeadChunkRows(
|
|
|
184
280
|
}
|
|
185
281
|
|
|
186
282
|
|
|
187
|
-
export function extractLmHeadChunk(
|
|
283
|
+
export async function extractLmHeadChunk(
|
|
188
284
|
data,
|
|
189
285
|
layout,
|
|
190
286
|
hiddenSize,
|
|
191
287
|
vocabSize,
|
|
192
288
|
rowOffset,
|
|
193
|
-
rowCount
|
|
289
|
+
rowCount,
|
|
290
|
+
sourceDtype = 'f32'
|
|
194
291
|
) {
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
292
|
+
const normalizedSourceDtype = String(sourceDtype ?? 'f32').toLowerCase();
|
|
293
|
+
if (normalizedSourceDtype !== 'f32' && normalizedSourceDtype !== 'f16' && normalizedSourceDtype !== 'bf16') {
|
|
294
|
+
throw new Error(`[Logits] Unsupported CPU LM head source dtype "${sourceDtype}".`);
|
|
295
|
+
}
|
|
296
|
+
if (!isRangeBackedCpuWeightSource(data)) {
|
|
297
|
+
return readTypedLmHeadChunk(
|
|
298
|
+
data,
|
|
299
|
+
layout,
|
|
300
|
+
hiddenSize,
|
|
301
|
+
vocabSize,
|
|
302
|
+
rowOffset,
|
|
303
|
+
rowCount,
|
|
304
|
+
normalizedSourceDtype
|
|
305
|
+
);
|
|
198
306
|
}
|
|
199
307
|
|
|
308
|
+
const bytesPerElement = normalizedSourceDtype === 'f32' ? 4 : 2;
|
|
200
309
|
const chunk = new Float32Array(hiddenSize * rowCount);
|
|
201
|
-
|
|
202
|
-
const
|
|
203
|
-
const
|
|
204
|
-
|
|
310
|
+
if (layout === 'row') {
|
|
311
|
+
const byteOffset = rowOffset * hiddenSize * bytesPerElement;
|
|
312
|
+
const byteLength = rowCount * hiddenSize * bytesPerElement;
|
|
313
|
+
const bytes = normalizeRangeBytes(
|
|
314
|
+
await data.loadRange(byteOffset, byteLength),
|
|
315
|
+
'CPU LM head range source'
|
|
316
|
+
);
|
|
317
|
+
if (bytes.byteLength !== byteLength) {
|
|
318
|
+
throw new Error(
|
|
319
|
+
`[Logits] CPU LM head range source returned ${bytes.byteLength} bytes, expected ${byteLength}.`
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
decodeChunkIntoOutput(bytes, normalizedSourceDtype, chunk, 0, rowCount * hiddenSize);
|
|
323
|
+
return chunk;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
for (let k = 0; k < hiddenSize; k += 1) {
|
|
327
|
+
const byteOffset = (k * vocabSize + rowOffset) * bytesPerElement;
|
|
328
|
+
const byteLength = rowCount * bytesPerElement;
|
|
329
|
+
const bytes = normalizeRangeBytes(
|
|
330
|
+
await data.loadRange(byteOffset, byteLength),
|
|
331
|
+
`CPU LM head range source column ${k}`
|
|
332
|
+
);
|
|
333
|
+
if (bytes.byteLength !== byteLength) {
|
|
334
|
+
throw new Error(
|
|
335
|
+
`[Logits] CPU LM head range source returned ${bytes.byteLength} bytes for column ${k}, expected ${byteLength}.`
|
|
336
|
+
);
|
|
337
|
+
}
|
|
338
|
+
decodeChunkIntoOutput(bytes, normalizedSourceDtype, chunk, k * rowCount, rowCount);
|
|
205
339
|
}
|
|
206
340
|
return chunk;
|
|
207
341
|
}
|
|
@@ -271,13 +405,14 @@ export async function computeChunkedLogitsGPU(
|
|
|
271
405
|
|
|
272
406
|
for (let rowOffset = 0; rowOffset < vocabSize; rowOffset += chunkRows) {
|
|
273
407
|
const rowCount = Math.min(chunkRows, vocabSize - rowOffset);
|
|
274
|
-
const chunkData = extractLmHeadChunk(
|
|
408
|
+
const chunkData = await extractLmHeadChunk(
|
|
275
409
|
lmHead.data,
|
|
276
410
|
lmHead.layout,
|
|
277
411
|
hiddenSize,
|
|
278
412
|
weightVocabSize,
|
|
279
413
|
rowOffset,
|
|
280
|
-
rowCount
|
|
414
|
+
rowCount,
|
|
415
|
+
lmHead.dtype
|
|
281
416
|
);
|
|
282
417
|
|
|
283
418
|
const f32Buffer = acquireBuffer(chunkData.byteLength, undefined, 'lm_head_chunk_f32');
|
|
@@ -419,6 +554,7 @@ export async function computeLogitsGPU(
|
|
|
419
554
|
await runProbes('pre_final_norm', inputBuffer, {
|
|
420
555
|
numTokens,
|
|
421
556
|
hiddenSize,
|
|
557
|
+
probes: config.debugProbes ?? null,
|
|
422
558
|
operatorDiagnostics,
|
|
423
559
|
dtype: inputDtype,
|
|
424
560
|
});
|
|
@@ -477,6 +613,7 @@ export async function computeLogitsGPU(
|
|
|
477
613
|
await runProbes('final_norm', finalNormTensor.buffer, {
|
|
478
614
|
numTokens,
|
|
479
615
|
hiddenSize,
|
|
616
|
+
probes: config.debugProbes ?? null,
|
|
480
617
|
operatorDiagnostics,
|
|
481
618
|
dtype: finalNormTensor.dtype,
|
|
482
619
|
});
|
|
@@ -23,7 +23,7 @@ export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } fro
|
|
|
23
23
|
export { computeLogitsGPU, recordLogitsGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
|
|
24
24
|
|
|
25
25
|
// Re-export utilities
|
|
26
|
-
export { extractLastPositionLogits, finalizeLogits } from './utils.js';
|
|
26
|
+
export { extractLastPositionLogits, finalizeLogits, readBufferWithCleanup } from './utils.js';
|
|
27
27
|
|
|
28
28
|
export interface ComputeLogitsOptions {
|
|
29
29
|
lastPositionOnly?: boolean;
|
|
@@ -48,12 +48,13 @@ export function activateKernelPathState(
|
|
|
48
48
|
kernelPathState: KernelPathResolutionResult | null | undefined
|
|
49
49
|
): void;
|
|
50
50
|
|
|
51
|
-
export function resolveAndActivateKernelPath(
|
|
52
|
-
options: KernelPathResolutionOptions
|
|
53
|
-
): KernelPathResolutionResult;
|
|
54
|
-
|
|
55
51
|
export function initTokenizerFromManifest(
|
|
56
52
|
manifest: Manifest,
|
|
57
53
|
baseUrl?: string | null,
|
|
58
54
|
storageContext?: PipelineStorageContext | null
|
|
59
55
|
): Promise<Tokenizer>;
|
|
56
|
+
|
|
57
|
+
export function assertManifestComputeLaneBinding(options: {
|
|
58
|
+
manifest: Manifest;
|
|
59
|
+
runtimeConfig: RuntimeConfigSchema;
|
|
60
|
+
}): void;
|
|
@@ -208,7 +208,7 @@ export function applyModelBatchingRuntimeDefaults(runtimeConfig, manifest, model
|
|
|
208
208
|
runtimeConfig,
|
|
209
209
|
manifest,
|
|
210
210
|
runtimeOverrides,
|
|
211
|
-
|
|
211
|
+
runtimeOverrides != null
|
|
212
212
|
);
|
|
213
213
|
if (!patch) {
|
|
214
214
|
return runtimeConfig;
|
|
@@ -266,10 +266,29 @@ export function applyModelBatchingRuntimeDefaults(runtimeConfig, manifest, model
|
|
|
266
266
|
dl.readbackMode = resolvedReadbackMode;
|
|
267
267
|
patch.batching.readbackMode = resolvedReadbackMode;
|
|
268
268
|
|
|
269
|
+
// Promote the resolved session and manifest largeWeights into runtimeConfig
|
|
270
|
+
// so existing pipeline reads of getRuntimeConfig().inference.session.* and
|
|
271
|
+
// .largeWeights.* pick up per-model values. buildResolvedDecodeLoopRuntimePatch()
|
|
272
|
+
// already applied runtime-over-manifest session precedence field-by-field.
|
|
273
|
+
// For largeWeights, an explicit runtime array wins, including [] to disable
|
|
274
|
+
// manifest gpu-resident overrides on constrained verification hosts.
|
|
275
|
+
const manifestInf = manifest?.inference ?? {};
|
|
276
|
+
const manifestLargeWeights = manifestInf.largeWeights;
|
|
277
|
+
const runtimeLargeWeightsOverrides =
|
|
278
|
+
runtimeConfig?.inference?.largeWeights?.gpuResidentOverrides;
|
|
279
|
+
const hasRuntimeLargeWeightsOverride = Array.isArray(runtimeLargeWeightsOverrides);
|
|
280
|
+
const largeWeightsPatch = (!hasRuntimeLargeWeightsOverride
|
|
281
|
+
&& manifestLargeWeights && typeof manifestLargeWeights === 'object'
|
|
282
|
+
&& Array.isArray(manifestLargeWeights.gpuResidentOverrides)
|
|
283
|
+
&& manifestLargeWeights.gpuResidentOverrides.length > 0)
|
|
284
|
+
? { gpuResidentOverrides: manifestLargeWeights.gpuResidentOverrides }
|
|
285
|
+
: null;
|
|
286
|
+
|
|
269
287
|
const nextRuntimeConfig = mergeRuntimeValues(runtimeConfig, {
|
|
270
288
|
inference: {
|
|
271
289
|
session: patch.session,
|
|
272
290
|
batching: patch.batching,
|
|
291
|
+
...(largeWeightsPatch ? { largeWeights: largeWeightsPatch } : {}),
|
|
273
292
|
},
|
|
274
293
|
});
|
|
275
294
|
log.info(
|
|
@@ -425,8 +444,6 @@ function summarizeUnsupportedKernelUsages(usages) {
|
|
|
425
444
|
}
|
|
426
445
|
|
|
427
446
|
function assertKernelPathFeatureCompatibility(
|
|
428
|
-
configuredKernelPathRef,
|
|
429
|
-
effectiveKernelPathRef,
|
|
430
447
|
resolvedKernelPath,
|
|
431
448
|
kernelPathSource,
|
|
432
449
|
capabilities,
|
|
@@ -439,16 +456,13 @@ function assertKernelPathFeatureCompatibility(
|
|
|
439
456
|
const policyAllowsSource = kernelPathPolicy.mode === 'capability-aware'
|
|
440
457
|
&& sourceScope.includes(kernelPathSource);
|
|
441
458
|
const remapRequested = policyAllowsSource && kernelPathPolicy.onIncompatible === 'remap';
|
|
442
|
-
const remapApplied = typeof configuredKernelPathRef === 'string'
|
|
443
|
-
&& typeof effectiveKernelPathRef === 'string'
|
|
444
|
-
&& configuredKernelPathRef !== effectiveKernelPathRef;
|
|
445
459
|
const summary = summarizeUnsupportedKernelUsages(unsupportedUsages);
|
|
446
460
|
|
|
447
|
-
if (remapRequested
|
|
461
|
+
if (remapRequested) {
|
|
448
462
|
throw new Error(
|
|
449
463
|
`KernelPath "${resolvedKernelPath?.id ?? 'unknown'}" requires unsupported GPU features (${summary}) ` +
|
|
450
|
-
`
|
|
451
|
-
'
|
|
464
|
+
`for source "${kernelPathSource}". String registry remaps are removed; use execution graph ` +
|
|
465
|
+
'capability transforms or choose a compatible inline kernelPath.'
|
|
452
466
|
);
|
|
453
467
|
}
|
|
454
468
|
|
|
@@ -545,6 +559,57 @@ function describeExplicitRuntimeDtypeMismatch(contract, explicitRuntime) {
|
|
|
545
559
|
return mismatches;
|
|
546
560
|
}
|
|
547
561
|
|
|
562
|
+
/**
|
|
563
|
+
* Manifest-side lane binding: the manifest variant tag claims a compute lane
|
|
564
|
+
* via quantizationInfo.compute. Once all runtime resolution phases have run,
|
|
565
|
+
* the resolved session must dispatch that same lane. If they disagree, the
|
|
566
|
+
* operator picked the wrong manifest variant or the wrong runtime profile.
|
|
567
|
+
*
|
|
568
|
+
* The compute lane covers activation/math/accum dtypes only. KV cache dtype
|
|
569
|
+
* is an orthogonal axis — Gemma-family layouts pair f32 compute with f16 KV
|
|
570
|
+
* for memory savings, and that combination is supported by design — so kv
|
|
571
|
+
* dtype is intentionally excluded from the comparison.
|
|
572
|
+
*
|
|
573
|
+
* Throws on mismatch. Returns silently when the manifest does not declare
|
|
574
|
+
* a compute lane (legacy / vision-only manifests) or when no resolved values
|
|
575
|
+
* are available to compare against.
|
|
576
|
+
*
|
|
577
|
+
* @param {Object} options
|
|
578
|
+
* @param {Object} options.manifest
|
|
579
|
+
* @param {Object} options.runtimeConfig
|
|
580
|
+
* @returns {void}
|
|
581
|
+
*/
|
|
582
|
+
export function assertManifestComputeLaneBinding({ manifest, runtimeConfig }) {
|
|
583
|
+
const declared = normalizeKernelDtype(manifest?.quantizationInfo?.compute);
|
|
584
|
+
if (!declared) return;
|
|
585
|
+
|
|
586
|
+
const session = runtimeConfig?.inference?.session ?? {};
|
|
587
|
+
const computeDefaults = session.compute?.defaults ?? {};
|
|
588
|
+
const candidates = [
|
|
589
|
+
['session.compute.defaults.activationDtype', computeDefaults.activationDtype],
|
|
590
|
+
['session.compute.defaults.mathDtype', computeDefaults.mathDtype],
|
|
591
|
+
['session.compute.defaults.accumDtype', computeDefaults.accumDtype],
|
|
592
|
+
];
|
|
593
|
+
|
|
594
|
+
const mismatches = [];
|
|
595
|
+
for (const [field, value] of candidates) {
|
|
596
|
+
const resolved = normalizeKernelDtype(value);
|
|
597
|
+
if (resolved && resolved !== declared) {
|
|
598
|
+
mismatches.push(`${field}=${value}`);
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
if (mismatches.length === 0) return;
|
|
602
|
+
|
|
603
|
+
throw new Error(
|
|
604
|
+
`Manifest "${manifest?.modelId ?? 'unknown'}" declares ` +
|
|
605
|
+
`quantizationInfo.compute=${declared} but runtime resolved ` +
|
|
606
|
+
`[${mismatches.join('; ')}]. ` +
|
|
607
|
+
'The manifest variant tag is the lane identity — load the manifest variant ' +
|
|
608
|
+
'whose compute lane matches the runtime profile, or pick a runtime profile ' +
|
|
609
|
+
'whose dtype defaults match this manifest.'
|
|
610
|
+
);
|
|
611
|
+
}
|
|
612
|
+
|
|
548
613
|
function assertManifestKernelPathDtypeCompatibility(manifest, resolvedKernelPath, kernelPathSource) {
|
|
549
614
|
if (!resolvedKernelPath) return;
|
|
550
615
|
if (kernelPathSource === 'config') return;
|
|
@@ -748,8 +813,6 @@ export function resolveKernelPathState(options) {
|
|
|
748
813
|
|
|
749
814
|
if (capabilities) {
|
|
750
815
|
assertKernelPathFeatureCompatibility(
|
|
751
|
-
configuredKernelPathRef,
|
|
752
|
-
configuredKernelPathRef,
|
|
753
816
|
resolvedKernelPath,
|
|
754
817
|
kernelPathSource,
|
|
755
818
|
capabilities,
|
|
@@ -790,12 +853,6 @@ export function activateKernelPathState(kernelPathState) {
|
|
|
790
853
|
);
|
|
791
854
|
}
|
|
792
855
|
|
|
793
|
-
export function resolveAndActivateKernelPath(options) {
|
|
794
|
-
const state = resolveKernelPathState(options);
|
|
795
|
-
activateKernelPathState(state);
|
|
796
|
-
return state;
|
|
797
|
-
}
|
|
798
|
-
|
|
799
856
|
export async function initTokenizerFromManifest(manifest, baseUrl, storageContext = null) {
|
|
800
857
|
return initTokenizer(manifest, {
|
|
801
858
|
baseUrl: baseUrl ?? undefined,
|
|
@@ -21,8 +21,6 @@ import type {
|
|
|
21
21
|
import type { LayerWeights } from './types.js';
|
|
22
22
|
import type { LoRAAdapter } from './lora.js';
|
|
23
23
|
|
|
24
|
-
export function isDecodeBuffer(decodeBuffers: DecodeBufferManager | null | undefined, buffer: GPUBuffer): boolean;
|
|
25
|
-
|
|
26
24
|
export function releaseOrTrack(
|
|
27
25
|
recorder: CommandRecorder | undefined,
|
|
28
26
|
buffer: GPUBuffer,
|
|
@@ -179,3 +177,16 @@ export function doAttention(
|
|
|
179
177
|
recorder?: CommandRecorder,
|
|
180
178
|
lora?: LoRAAdapter | null
|
|
181
179
|
): Promise<AttentionResult>;
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Initialize per-layer convolution state from the loaded weight buffer.
|
|
183
|
+
* Resolves kernel shape, dtype, and input projection bindings.
|
|
184
|
+
*/
|
|
185
|
+
export declare function initConvLayerState(
|
|
186
|
+
convState: Record<string, unknown>,
|
|
187
|
+
convKernel: GPUBuffer | WeightBuffer | Float32Array | ArrayBuffer | CpuWeightBuffer,
|
|
188
|
+
convInProj: GPUBuffer | WeightBuffer | Float32Array | ArrayBuffer | CpuWeightBuffer | null,
|
|
189
|
+
hiddenSize: number,
|
|
190
|
+
label: string,
|
|
191
|
+
layerIdx: number
|
|
192
|
+
): Promise<void>;
|
|
@@ -25,7 +25,7 @@ import { runLinearAttentionLayer } from './linear-attention.js';
|
|
|
25
25
|
import { runGatedShortConvGPU } from '../../../gpu/kernels/gated-short-conv.js';
|
|
26
26
|
|
|
27
27
|
|
|
28
|
-
|
|
28
|
+
function isDecodeBuffer(decodeBuffers, buffer) {
|
|
29
29
|
return !!decodeBuffers?.ownsBuffer(buffer);
|
|
30
30
|
}
|
|
31
31
|
|