doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -4,6 +4,8 @@ import { initDevice, getDevice, getKernelCapabilities } from '../gpu/device.js';
|
|
|
4
4
|
import { parseManifest, getExpectedShardHash } from '../formats/rdrr/index.js';
|
|
5
5
|
import { createPipeline } from './pipelines/text.js';
|
|
6
6
|
import { createNodeFileShardStorageContext } from './pipelines/text/init.js';
|
|
7
|
+
import { createHttpArtifactStorageContext } from '../storage/artifact-storage-context.js';
|
|
8
|
+
import { resolveManifestArtifactSource } from '../client/runtime/model-source.js';
|
|
7
9
|
import { log as debugLog } from '../debug/index.js';
|
|
8
10
|
import { getRuntimeConfig, setRuntimeConfig } from '../config/runtime.js';
|
|
9
11
|
|
|
@@ -144,8 +146,37 @@ function toShardBytes(buffer, shardIndex) {
|
|
|
144
146
|
throw new Error(`Shard ${shardIndex} did not return an in-memory buffer`);
|
|
145
147
|
}
|
|
146
148
|
|
|
149
|
+
function createInitializationPhaseError(error, phase, context = {}) {
|
|
150
|
+
const message = error?.message || String(error);
|
|
151
|
+
const wrapped = new Error(
|
|
152
|
+
`Inference initialization phase "${phase}" failed: ${message}`,
|
|
153
|
+
error instanceof Error ? { cause: error } : undefined
|
|
154
|
+
);
|
|
155
|
+
wrapped.name = error?.name || 'Error';
|
|
156
|
+
if (error?.code !== undefined) {
|
|
157
|
+
wrapped.code = error.code;
|
|
158
|
+
}
|
|
159
|
+
wrapped.details = {
|
|
160
|
+
...(error?.details && typeof error.details === 'object' ? error.details : {}),
|
|
161
|
+
initializationPhase: phase,
|
|
162
|
+
...context,
|
|
163
|
+
};
|
|
164
|
+
return wrapped;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
async function withInitializationPhase(phase, context, run) {
|
|
168
|
+
try {
|
|
169
|
+
return await run();
|
|
170
|
+
} catch (error) {
|
|
171
|
+
if (error?.details?.initializationPhase) {
|
|
172
|
+
throw error;
|
|
173
|
+
}
|
|
174
|
+
throw createInitializationPhaseError(error, phase, context);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
147
177
|
|
|
148
|
-
|
|
178
|
+
|
|
179
|
+
function createHttpShardLoader(baseUrl, manifest, log) {
|
|
149
180
|
const algorithm = manifest.hashAlgorithm;
|
|
150
181
|
if (!algorithm) {
|
|
151
182
|
throw new Error('Manifest missing hashAlgorithm for shard delivery.');
|
|
@@ -217,11 +248,29 @@ export function createHttpShardLoader(baseUrl, manifest, log) {
|
|
|
217
248
|
};
|
|
218
249
|
}
|
|
219
250
|
|
|
220
|
-
export function createHarnessShardStorageContext(modelUrl, manifest, log) {
|
|
251
|
+
export function createHarnessShardStorageContext(modelUrl, manifest, log, options = {}) {
|
|
221
252
|
const nodeFileStorageContext = createNodeFileShardStorageContext(modelUrl, manifest);
|
|
222
253
|
if (nodeFileStorageContext) {
|
|
223
254
|
return nodeFileStorageContext;
|
|
224
255
|
}
|
|
256
|
+
const runtimeConfig = getRuntimeConfig();
|
|
257
|
+
const sourceOrder = runtimeConfig?.loading?.distribution?.sourceOrder;
|
|
258
|
+
const httpOnly = Array.isArray(sourceOrder)
|
|
259
|
+
&& sourceOrder.length === 1
|
|
260
|
+
&& String(sourceOrder[0]).trim().toLowerCase() === 'http';
|
|
261
|
+
const explicitHttpLoadMode = String(options.loadMode ?? '').trim().toLowerCase() === 'http';
|
|
262
|
+
if (explicitHttpLoadMode || httpOnly) {
|
|
263
|
+
const httpContext = createHttpArtifactStorageContext(modelUrl, manifest, {
|
|
264
|
+
verifyHashes: runtimeConfig?.loading?.shardCache?.verifyHashes === true,
|
|
265
|
+
rangeCacheBlockBytes: runtimeConfig?.loading?.shardCache?.rangeCacheBlockBytes,
|
|
266
|
+
rangeCacheMaxBytes: runtimeConfig?.loading?.shardCache?.rangeCacheMaxBytes,
|
|
267
|
+
rangeCacheMinBytes: runtimeConfig?.loading?.shardCache?.rangeCacheMinBytes,
|
|
268
|
+
});
|
|
269
|
+
if (httpContext) {
|
|
270
|
+
log?.('Using HTTP range artifact storage context');
|
|
271
|
+
return httpContext;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
225
274
|
return {
|
|
226
275
|
loadShard: createHttpShardLoader(modelUrl, manifest, log),
|
|
227
276
|
};
|
|
@@ -232,18 +281,16 @@ export function createHarnessShardStorageContext(modelUrl, manifest, log) {
|
|
|
232
281
|
// ============================================================================
|
|
233
282
|
|
|
234
283
|
|
|
235
|
-
|
|
284
|
+
async function fetchManifestPayload(manifestUrl) {
|
|
236
285
|
const response = await fetch(manifestUrl);
|
|
237
286
|
if (!response.ok) {
|
|
238
287
|
throw new Error(`Failed to fetch manifest: ${response.status}`);
|
|
239
288
|
}
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
await initDevice();
|
|
246
|
-
return getKernelCapabilities();
|
|
289
|
+
const text = await response.text();
|
|
290
|
+
return {
|
|
291
|
+
text,
|
|
292
|
+
manifest: parseManifest(text),
|
|
293
|
+
};
|
|
247
294
|
}
|
|
248
295
|
|
|
249
296
|
|
|
@@ -297,7 +344,11 @@ export async function initializeInference(modelUrl, options = {}) {
|
|
|
297
344
|
onProgress('init', 0, 'Initializing WebGPU...');
|
|
298
345
|
log('Initializing WebGPU...');
|
|
299
346
|
|
|
300
|
-
await
|
|
347
|
+
await withInitializationPhase(
|
|
348
|
+
'initDevice',
|
|
349
|
+
{ modelUrl },
|
|
350
|
+
() => initDevice()
|
|
351
|
+
);
|
|
301
352
|
const device = getDevice();
|
|
302
353
|
const capabilities = getKernelCapabilities();
|
|
303
354
|
|
|
@@ -308,7 +359,24 @@ export async function initializeInference(modelUrl, options = {}) {
|
|
|
308
359
|
log('Fetching manifest...');
|
|
309
360
|
|
|
310
361
|
const manifestUrl = `${modelUrl}/manifest.json`;
|
|
311
|
-
const
|
|
362
|
+
const manifestPayload = await withInitializationPhase(
|
|
363
|
+
'fetchManifest',
|
|
364
|
+
{ modelUrl, manifestUrl },
|
|
365
|
+
() => fetchManifestPayload(manifestUrl)
|
|
366
|
+
);
|
|
367
|
+
const artifactSource = await withInitializationPhase(
|
|
368
|
+
'resolveArtifactSource',
|
|
369
|
+
{ modelUrl, manifestUrl },
|
|
370
|
+
() => resolveManifestArtifactSource({
|
|
371
|
+
modelId: options.modelId || manifestPayload.manifest?.modelId || modelUrl,
|
|
372
|
+
baseUrl: modelUrl,
|
|
373
|
+
manifest: null,
|
|
374
|
+
trace: [],
|
|
375
|
+
}, manifestPayload)
|
|
376
|
+
);
|
|
377
|
+
const manifest = artifactSource.manifest;
|
|
378
|
+
const storageManifest = artifactSource.storageManifest ?? manifest;
|
|
379
|
+
const storageModelUrl = artifactSource.storageBaseUrl ?? modelUrl;
|
|
312
380
|
|
|
313
381
|
if (intentBundleConfig.enabled && intentBundleConfig.bundleUrl) {
|
|
314
382
|
const intentBundleModule = await loadIntentBundleModule();
|
|
@@ -339,7 +407,9 @@ export async function initializeInference(modelUrl, options = {}) {
|
|
|
339
407
|
log(`Model: ${modelLabel}`);
|
|
340
408
|
|
|
341
409
|
// 3. Create shard loader
|
|
342
|
-
const storageContext = createHarnessShardStorageContext(
|
|
410
|
+
const storageContext = createHarnessShardStorageContext(storageModelUrl, storageManifest, log, {
|
|
411
|
+
loadMode: options.loadMode ?? null,
|
|
412
|
+
});
|
|
343
413
|
|
|
344
414
|
// 4. Build runtime options
|
|
345
415
|
|
|
@@ -351,16 +421,20 @@ export async function initializeInference(modelUrl, options = {}) {
|
|
|
351
421
|
onProgress('pipeline', 0.2, 'Creating pipeline...');
|
|
352
422
|
log('Creating pipeline...');
|
|
353
423
|
|
|
354
|
-
const pipeline = await
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
424
|
+
const pipeline = await withInitializationPhase(
|
|
425
|
+
'createPipeline',
|
|
426
|
+
{ modelUrl, modelId: manifest.modelId ?? null },
|
|
427
|
+
() => createPipeline( ( (manifest)), {
|
|
428
|
+
storage: storageContext,
|
|
429
|
+
gpu: { device },
|
|
430
|
+
runtime,
|
|
431
|
+
baseUrl: storageModelUrl,
|
|
432
|
+
onProgress: ( progress) => {
|
|
433
|
+
const pct = 0.2 + progress.percent * 0.8;
|
|
434
|
+
onProgress(progress.stage || 'loading', pct, progress.message);
|
|
435
|
+
},
|
|
436
|
+
})
|
|
437
|
+
);
|
|
364
438
|
|
|
365
439
|
onProgress('complete', 1, 'Ready');
|
|
366
440
|
log('Pipeline ready');
|
|
@@ -5,6 +5,7 @@ import { BaseTokenizer } from './tokenizers/base.js';
|
|
|
5
5
|
import { TransformersTokenizer, BundledTokenizer } from './tokenizers/bundled.js';
|
|
6
6
|
import { SentencePieceTokenizer } from './tokenizers/sentencepiece.js';
|
|
7
7
|
import { BPETokenizer } from './tokenizers/bpe.js';
|
|
8
|
+
import { toArrayBuffer } from '../utils/array-buffer.js';
|
|
8
9
|
|
|
9
10
|
const BUNDLED_TOKENIZER_CACHE = new Map();
|
|
10
11
|
|
|
@@ -12,16 +13,6 @@ function hasScheme(value) {
|
|
|
12
13
|
return /^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(value);
|
|
13
14
|
}
|
|
14
15
|
|
|
15
|
-
function toArrayBuffer(value, label) {
|
|
16
|
-
if (value instanceof ArrayBuffer) {
|
|
17
|
-
return value;
|
|
18
|
-
}
|
|
19
|
-
if (value instanceof Uint8Array) {
|
|
20
|
-
return value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength);
|
|
21
|
-
}
|
|
22
|
-
throw new Error(`${label} must return ArrayBuffer or Uint8Array.`);
|
|
23
|
-
}
|
|
24
|
-
|
|
25
16
|
function parseTokenizerJsonPayload(value) {
|
|
26
17
|
if (value == null) {
|
|
27
18
|
return null;
|
|
@@ -23,6 +23,8 @@ export class SentencePieceTokenizer extends BaseTokenizer {
|
|
|
23
23
|
|
|
24
24
|
#unkId = 0;
|
|
25
25
|
|
|
26
|
+
#addDummyPrefix = true;
|
|
27
|
+
|
|
26
28
|
|
|
27
29
|
constructor(config = {}) {
|
|
28
30
|
// SentencePieceTokenizer gets vocabSize from load(), so defer validation
|
|
@@ -38,6 +40,7 @@ export class SentencePieceTokenizer extends BaseTokenizer {
|
|
|
38
40
|
this.#algorithm = 'unigram';
|
|
39
41
|
this.#byteTokens.clear();
|
|
40
42
|
this.vocabSize = 0;
|
|
43
|
+
this.#addDummyPrefix = true;
|
|
41
44
|
}
|
|
42
45
|
|
|
43
46
|
|
|
@@ -76,9 +79,10 @@ export class SentencePieceTokenizer extends BaseTokenizer {
|
|
|
76
79
|
|
|
77
80
|
|
|
78
81
|
async #parseModelProto(buffer) {
|
|
79
|
-
const view = new DataView(buffer);
|
|
80
82
|
const bytes = new Uint8Array(buffer);
|
|
81
83
|
let offset = 0;
|
|
84
|
+
const standardPieces = [];
|
|
85
|
+
const liteRTPieceCandidates = [];
|
|
82
86
|
|
|
83
87
|
// SentencePiece model is a protobuf with:
|
|
84
88
|
// - Field 1: trainer_spec
|
|
@@ -94,10 +98,21 @@ export class SentencePieceTokenizer extends BaseTokenizer {
|
|
|
94
98
|
const wireType = tag & 0x7;
|
|
95
99
|
|
|
96
100
|
if (fieldNumber === 1 || fieldNumber === 2) {
|
|
97
|
-
// Skip trainer_spec and normalizer_spec
|
|
101
|
+
// Skip trainer_spec and normalizer_spec in standard ModelProto files.
|
|
102
|
+
// LiteRT task metadata may store a compact repeated SentencePiece list
|
|
103
|
+
// at field 1; collect candidates and choose it only if field 3 did not
|
|
104
|
+
// expose the richer standard piece list.
|
|
98
105
|
if (wireType === 2) {
|
|
99
106
|
const { value: length, newOffset } = this.#readVarint(bytes, offset);
|
|
100
|
-
offset = newOffset
|
|
107
|
+
offset = newOffset;
|
|
108
|
+
if (fieldNumber === 1) {
|
|
109
|
+
const pieceData = bytes.slice(offset, offset + length);
|
|
110
|
+
const candidate = this.#decodePiece(pieceData);
|
|
111
|
+
if (candidate && (candidate.hasScore || candidate.hasType)) {
|
|
112
|
+
liteRTPieceCandidates.push(candidate);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
offset += length;
|
|
101
116
|
}
|
|
102
117
|
} else if (fieldNumber === 3 && wireType === 2) {
|
|
103
118
|
// SentencePiece entry
|
|
@@ -105,7 +120,10 @@ export class SentencePieceTokenizer extends BaseTokenizer {
|
|
|
105
120
|
offset = newOffset;
|
|
106
121
|
|
|
107
122
|
const pieceData = bytes.slice(offset, offset + length);
|
|
108
|
-
this.#
|
|
123
|
+
const piece = this.#decodePiece(pieceData);
|
|
124
|
+
if (piece) {
|
|
125
|
+
standardPieces.push(piece);
|
|
126
|
+
}
|
|
109
127
|
offset += length;
|
|
110
128
|
} else {
|
|
111
129
|
// Skip unknown field
|
|
@@ -125,6 +143,12 @@ export class SentencePieceTokenizer extends BaseTokenizer {
|
|
|
125
143
|
}
|
|
126
144
|
}
|
|
127
145
|
|
|
146
|
+
const isLiteRTPieceList = liteRTPieceCandidates.length > standardPieces.length;
|
|
147
|
+
const selectedPieces = isLiteRTPieceList ? liteRTPieceCandidates : standardPieces;
|
|
148
|
+
for (const [id, piece] of selectedPieces.entries()) {
|
|
149
|
+
this.#addPiece(piece, id);
|
|
150
|
+
}
|
|
151
|
+
|
|
128
152
|
// Set up special tokens
|
|
129
153
|
if (this.specialTokens.unk == null) {
|
|
130
154
|
throw new Error('[Tokenizer] unk token is required for SentencePiece.');
|
|
@@ -134,15 +158,25 @@ export class SentencePieceTokenizer extends BaseTokenizer {
|
|
|
134
158
|
// Determine algorithm from model characteristics
|
|
135
159
|
// (Unigram has scores, BPE typically doesn't)
|
|
136
160
|
const hasScores = [...this.#pieces.values()].some(p => p.score !== 0);
|
|
137
|
-
|
|
161
|
+
const tokenizerRuleContext = { hasScores, isLiteRTPieceList };
|
|
162
|
+
this.#algorithm = selectRuleValue('converter', 'tokenizer', 'type', tokenizerRuleContext);
|
|
163
|
+
this.#addDummyPrefix = selectRuleValue(
|
|
164
|
+
'converter',
|
|
165
|
+
'tokenizer',
|
|
166
|
+
'addDummyPrefix',
|
|
167
|
+
tokenizerRuleContext
|
|
168
|
+
);
|
|
138
169
|
}
|
|
139
170
|
|
|
140
171
|
|
|
141
|
-
#
|
|
172
|
+
#decodePiece(bytes) {
|
|
142
173
|
let offset = 0;
|
|
143
174
|
let piece = '';
|
|
175
|
+
let hasPiece = false;
|
|
144
176
|
let score = 0;
|
|
145
177
|
let type = 1; // NORMAL by default
|
|
178
|
+
let hasScore = false;
|
|
179
|
+
let hasType = false;
|
|
146
180
|
|
|
147
181
|
while (offset < bytes.length) {
|
|
148
182
|
const { value: tag, newOffset: tagOffset } = this.#readVarint(bytes, offset);
|
|
@@ -156,16 +190,19 @@ export class SentencePieceTokenizer extends BaseTokenizer {
|
|
|
156
190
|
const { value: length, newOffset } = this.#readVarint(bytes, offset);
|
|
157
191
|
offset = newOffset;
|
|
158
192
|
piece = new TextDecoder().decode(bytes.slice(offset, offset + length));
|
|
193
|
+
hasPiece = true;
|
|
159
194
|
offset += length;
|
|
160
195
|
} else if (fieldNumber === 2 && wireType === 5) {
|
|
161
196
|
// score (float32)
|
|
162
197
|
const view = new DataView(bytes.buffer, bytes.byteOffset + offset, 4);
|
|
163
198
|
score = view.getFloat32(0, true);
|
|
199
|
+
hasScore = true;
|
|
164
200
|
offset += 4;
|
|
165
201
|
} else if (fieldNumber === 3 && wireType === 0) {
|
|
166
202
|
// type (varint enum)
|
|
167
203
|
const { value, newOffset } = this.#readVarint(bytes, offset);
|
|
168
204
|
type = value;
|
|
205
|
+
hasType = true;
|
|
169
206
|
offset = newOffset;
|
|
170
207
|
} else {
|
|
171
208
|
// Skip unknown
|
|
@@ -181,15 +218,26 @@ export class SentencePieceTokenizer extends BaseTokenizer {
|
|
|
181
218
|
}
|
|
182
219
|
}
|
|
183
220
|
|
|
184
|
-
if (
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
221
|
+
if (!hasPiece) {
|
|
222
|
+
return null;
|
|
223
|
+
}
|
|
224
|
+
return { piece, score, type, hasScore, hasType };
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
#addPiece(entry, id) {
|
|
228
|
+
if (!entry?.piece) {
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
if (!this.#pieces.has(entry.piece)) {
|
|
232
|
+
this.#pieces.set(entry.piece, { id, score: entry.score, type: entry.type });
|
|
233
|
+
}
|
|
234
|
+
this.#reverseVocab.set(id, entry.piece);
|
|
235
|
+
this.vocabSize = Math.max(this.vocabSize, id + 1);
|
|
189
236
|
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
237
|
+
// Track byte tokens (▁ prefix tokens and <0xXX> byte tokens)
|
|
238
|
+
if (entry.piece.match(/^<0x[0-9A-Fa-f]{2}>$/)) {
|
|
239
|
+
const byteVal = parseInt(entry.piece.slice(3, 5), 16);
|
|
240
|
+
if (!this.#byteTokens.has(byteVal)) {
|
|
193
241
|
this.#byteTokens.set(byteVal, id);
|
|
194
242
|
}
|
|
195
243
|
}
|
|
@@ -255,7 +303,9 @@ export class SentencePieceTokenizer extends BaseTokenizer {
|
|
|
255
303
|
|
|
256
304
|
// Normalize: add sentence piece prefix (▁ for word start)
|
|
257
305
|
const normalized = text.replace(/ /g, '▁');
|
|
258
|
-
const prefixed =
|
|
306
|
+
const prefixed = this.#addDummyPrefix && !text.startsWith(' ')
|
|
307
|
+
? `▁${normalized}`
|
|
308
|
+
: normalized;
|
|
259
309
|
|
|
260
310
|
if (this.#algorithm === 'unigram') {
|
|
261
311
|
ids.push(...this.#encodeUnigram(prefixed));
|
|
@@ -145,24 +145,3 @@ export interface BundledTokenizerJson {
|
|
|
145
145
|
addSpacePrefix?: boolean;
|
|
146
146
|
}
|
|
147
147
|
|
|
148
|
-
/** Viterbi state for Unigram encoding */
|
|
149
|
-
export interface ViterbiState {
|
|
150
|
-
score: number;
|
|
151
|
-
prev: number;
|
|
152
|
-
tokenLen: number;
|
|
153
|
-
isBytes?: boolean;
|
|
154
|
-
bytes?: Uint8Array;
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
/** Special token pattern for matching */
|
|
158
|
-
export interface SpecialTokenPattern {
|
|
159
|
-
content: string;
|
|
160
|
-
id: number;
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
/** Text segment (plain text or special token) */
|
|
164
|
-
export interface TextSegment {
|
|
165
|
-
text?: string;
|
|
166
|
-
id?: number;
|
|
167
|
-
isSpecial: boolean;
|
|
168
|
-
}
|
|
@@ -97,6 +97,7 @@ export declare class DopplerLoader {
|
|
|
97
97
|
q4kLayout: 'row' | 'col' | null;
|
|
98
98
|
keepF32Weights: boolean;
|
|
99
99
|
q4kMaterializationMode: 'dense' | 'fused' | 'mixed';
|
|
100
|
+
q4kFusedRoles: string[];
|
|
100
101
|
|
|
101
102
|
constructor(loadingConfig?: LoadingConfigSchema);
|
|
102
103
|
|
|
@@ -112,6 +113,8 @@ export declare class DopplerLoader {
|
|
|
112
113
|
|
|
113
114
|
setTensorsJsonUrl(url: string | null): void;
|
|
114
115
|
|
|
116
|
+
setTensorsJsonLoader(loadTensorsJson: (() => Promise<string | Record<string, unknown> | null | undefined>) | null): void;
|
|
117
|
+
|
|
115
118
|
init(): Promise<void>;
|
|
116
119
|
|
|
117
120
|
setManifest(manifest: RDRRManifest | RuntimeModelContract): void;
|
|
@@ -40,6 +40,7 @@ import {
|
|
|
40
40
|
loadTensorToGPU,
|
|
41
41
|
loadTensorToCPU,
|
|
42
42
|
} from './tensors/tensor-loader.js';
|
|
43
|
+
import { annotateWeightLoadError } from '../inference/pipelines/text/load-errors.js';
|
|
43
44
|
import { loadEmbeddings } from './embedding-loader.js';
|
|
44
45
|
import { loadPerLayerInputWeights } from './per-layer-input-loader.js';
|
|
45
46
|
import { loadLayer } from './layer-loader.js';
|
|
@@ -64,6 +65,17 @@ function hasExpertGroups(manifest) {
|
|
|
64
65
|
return Object.keys(manifest.groups).some((groupId) => groupId.includes('.expert.'));
|
|
65
66
|
}
|
|
66
67
|
|
|
68
|
+
function isDirectSourceRuntimeManifest(manifest) {
|
|
69
|
+
const sourceRuntime = manifest?.metadata?.sourceRuntime;
|
|
70
|
+
const mode = typeof sourceRuntime?.mode === 'string'
|
|
71
|
+
? sourceRuntime.mode.trim().toLowerCase()
|
|
72
|
+
: '';
|
|
73
|
+
const sourceKind = typeof sourceRuntime?.sourceKind === 'string'
|
|
74
|
+
? sourceRuntime.sourceKind.trim().toLowerCase()
|
|
75
|
+
: '';
|
|
76
|
+
return mode === 'direct-source' && sourceKind !== 'rdrr';
|
|
77
|
+
}
|
|
78
|
+
|
|
67
79
|
/**
|
|
68
80
|
* Detect whether a manifest describes a Mixture-of-Experts model.
|
|
69
81
|
* Validates that manifests with expert groups also have moeConfig.
|
|
@@ -84,6 +96,25 @@ function isGpuBufferInstance(value) {
|
|
|
84
96
|
return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
|
|
85
97
|
}
|
|
86
98
|
|
|
99
|
+
function getTensorShardIndices(location) {
|
|
100
|
+
if (Array.isArray(location?.spans) && location.spans.length > 0) {
|
|
101
|
+
return Array.from(new Set(location.spans.map((span) => span?.shardIndex).filter(Number.isInteger)));
|
|
102
|
+
}
|
|
103
|
+
return Number.isInteger(location?.shardIndex) ? [location.shardIndex] : [];
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function annotateTensorLoadError(error, name, location, details = {}) {
|
|
107
|
+
return annotateWeightLoadError(error, {
|
|
108
|
+
tensorName: name,
|
|
109
|
+
tensorRole: location?.role ?? null,
|
|
110
|
+
tensorDtype: location?.dtype ?? null,
|
|
111
|
+
tensorShape: Array.isArray(location?.shape) ? [...location.shape] : null,
|
|
112
|
+
tensorSizeBytes: Number.isFinite(location?.size) ? location.size : null,
|
|
113
|
+
tensorShardIndices: getTensorShardIndices(location),
|
|
114
|
+
...details,
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
|
|
87
118
|
// Re-export types for backward compatibility
|
|
88
119
|
export {
|
|
89
120
|
// Types are in .d.ts file
|
|
@@ -168,6 +199,8 @@ export class DopplerLoader {
|
|
|
168
199
|
|
|
169
200
|
q4kMaterializationMode = 'dense';
|
|
170
201
|
|
|
202
|
+
q4kFusedRoles = [];
|
|
203
|
+
|
|
171
204
|
// Internal tracking
|
|
172
205
|
|
|
173
206
|
#normOffsetLogged = false;
|
|
@@ -177,6 +210,7 @@ export class DopplerLoader {
|
|
|
177
210
|
#memoryMonitor = null;
|
|
178
211
|
|
|
179
212
|
#tensorsJsonUrl = null;
|
|
213
|
+
#loadTensorsJson = null;
|
|
180
214
|
|
|
181
215
|
#loadShardOverride = null;
|
|
182
216
|
|
|
@@ -223,6 +257,9 @@ export class DopplerLoader {
|
|
|
223
257
|
this.q4kLayout = config.q4kLayout;
|
|
224
258
|
this.keepF32Weights = config.keepF32Weights;
|
|
225
259
|
this.q4kMaterializationMode = config.q4kMaterializationMode ?? 'dense';
|
|
260
|
+
this.q4kFusedRoles = Array.isArray(config.q4kFusedRoles)
|
|
261
|
+
? [...new Set(config.q4kFusedRoles.map((role) => String(role)).filter(Boolean))].sort()
|
|
262
|
+
: [];
|
|
226
263
|
}
|
|
227
264
|
|
|
228
265
|
|
|
@@ -285,6 +322,10 @@ export class DopplerLoader {
|
|
|
285
322
|
this.#tensorsJsonUrl = url;
|
|
286
323
|
}
|
|
287
324
|
|
|
325
|
+
setTensorsJsonLoader(loadTensorsJson) {
|
|
326
|
+
this.#loadTensorsJson = typeof loadTensorsJson === 'function' ? loadTensorsJson : null;
|
|
327
|
+
}
|
|
328
|
+
|
|
288
329
|
|
|
289
330
|
async #loadShard(shardIndex, options) {
|
|
290
331
|
return this.shardCache.load(shardIndex, options);
|
|
@@ -371,7 +412,8 @@ export class DopplerLoader {
|
|
|
371
412
|
|
|
372
413
|
|
|
373
414
|
#shouldStreamLargeWeight(name, location, label) {
|
|
374
|
-
|
|
415
|
+
const manifestOverrides = this.manifest?.inference?.largeWeights?.gpuResidentOverrides ?? null;
|
|
416
|
+
return shouldStreamLargeWeight(name, location, label, this.gpuCapabilities, this.keepF32Weights, manifestOverrides);
|
|
375
417
|
}
|
|
376
418
|
|
|
377
419
|
|
|
@@ -434,15 +476,14 @@ export class DopplerLoader {
|
|
|
434
476
|
log.warn('Loader', 'Dense model on discrete GPU - performance limited. Consider MoE model.');
|
|
435
477
|
}
|
|
436
478
|
|
|
437
|
-
if (
|
|
438
|
-
// Avoid a full re-hash on every warm load. Presence check is enough to
|
|
439
|
-
// decide "cached vs missing"; hash verification is performed at download/import time.
|
|
479
|
+
if (!this.shardCache.hasCustomLoader) {
|
|
440
480
|
const integrity = await verifyIntegrity({ checkHashes: false });
|
|
441
481
|
if (!integrity.valid) {
|
|
442
482
|
throw new Error(
|
|
443
|
-
`
|
|
483
|
+
`Artifact contract preflight failed for "${this.manifest?.modelId ?? modelId}". ` +
|
|
444
484
|
`Missing shards: ${integrity.missingShards.length}, ` +
|
|
445
|
-
`
|
|
485
|
+
`corrupt shards: ${integrity.corruptShards.length}. ` +
|
|
486
|
+
'Re-import, re-download, or provide a manifest with a valid weightsRef.'
|
|
446
487
|
);
|
|
447
488
|
}
|
|
448
489
|
}
|
|
@@ -667,6 +708,7 @@ export class DopplerLoader {
|
|
|
667
708
|
|
|
668
709
|
const locations = await buildTensorLocations(this.manifest, {
|
|
669
710
|
tensorsJsonUrl: this.#tensorsJsonUrl,
|
|
711
|
+
loadTensorsJson: this.#loadTensorsJson,
|
|
670
712
|
hasCustomLoader: this.shardCache.hasCustomLoader,
|
|
671
713
|
});
|
|
672
714
|
|
|
@@ -730,7 +772,20 @@ export class DopplerLoader {
|
|
|
730
772
|
// Range-capable custom loaders are expected to serve fine-grained tensor reads.
|
|
731
773
|
// Whole-shard prefetch defeats that contract and can force invalid >4 GiB reads
|
|
732
774
|
// for direct-source SafeTensors bundles.
|
|
733
|
-
if (this.shardCache.hasCustomLoader && this.shardCache.canStreamRanges)
|
|
775
|
+
if (this.shardCache.hasCustomLoader && this.shardCache.canStreamRanges) {
|
|
776
|
+
if (prefetch.allowRangeLoaderPrefetch !== true) return;
|
|
777
|
+
if (isDirectSourceRuntimeManifest(this.manifest)) {
|
|
778
|
+
throw new Error(
|
|
779
|
+
'runtime.loading.prefetch.allowRangeLoaderPrefetch is only supported for RDRR shard manifests; ' +
|
|
780
|
+
'direct-source artifacts must use range reads.'
|
|
781
|
+
);
|
|
782
|
+
}
|
|
783
|
+
if (!Array.isArray(this.manifest?.shards) || this.manifest.shards.length === 0) {
|
|
784
|
+
throw new Error(
|
|
785
|
+
'runtime.loading.prefetch.allowRangeLoaderPrefetch requires a manifest with explicit RDRR shards.'
|
|
786
|
+
);
|
|
787
|
+
}
|
|
788
|
+
}
|
|
734
789
|
|
|
735
790
|
const layersAhead = prefetch.layersAhead;
|
|
736
791
|
if (!Number.isFinite(layersAhead) || layersAhead <= 0) return;
|
|
@@ -769,9 +824,19 @@ export class DopplerLoader {
|
|
|
769
824
|
debugTrace.loader(`Loading ${name}: shape=${JSON.stringify(location.shape)}, size=${location.size}, dtype=${location.dtype}, spans=${!!location.spans}`);
|
|
770
825
|
}
|
|
771
826
|
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
827
|
+
const streamedUpload = toGPU && this.#shouldStreamUploadToGPU(location);
|
|
828
|
+
let shardData;
|
|
829
|
+
try {
|
|
830
|
+
shardData = streamedUpload
|
|
831
|
+
? await this.#assembleShardDataToGpuBuffer(location, name)
|
|
832
|
+
: await this.#assembleShardData(location, name);
|
|
833
|
+
} catch (error) {
|
|
834
|
+
throw annotateTensorLoadError(error, name, location, {
|
|
835
|
+
tensorLoadStage: streamedUpload ? 'streamShardToGpuBuffer' : 'assembleShardData',
|
|
836
|
+
toGPU,
|
|
837
|
+
streamedUpload,
|
|
838
|
+
});
|
|
839
|
+
}
|
|
775
840
|
|
|
776
841
|
if (toGPU) {
|
|
777
842
|
const device = getDevice();
|
|
@@ -792,6 +857,7 @@ export class DopplerLoader {
|
|
|
792
857
|
const config = {
|
|
793
858
|
useFusedQ4K: this.useFusedQ4K,
|
|
794
859
|
q4kMaterializationMode: this.q4kMaterializationMode,
|
|
860
|
+
q4kFusedRoles: this.q4kFusedRoles,
|
|
795
861
|
keepF32Weights: this.keepF32Weights,
|
|
796
862
|
q4kLayout: this.q4kLayout,
|
|
797
863
|
loaderDebug: this.#loaderDebug,
|
|
@@ -799,7 +865,19 @@ export class DopplerLoader {
|
|
|
799
865
|
allowF32UpcastNonMatmul,
|
|
800
866
|
};
|
|
801
867
|
|
|
802
|
-
|
|
868
|
+
let result;
|
|
869
|
+
try {
|
|
870
|
+
result = await loadTensorToGPU(shardData, location, name, config);
|
|
871
|
+
} catch (error) {
|
|
872
|
+
if (isGpuBufferInstance(shardData)) {
|
|
873
|
+
releaseBuffer(shardData);
|
|
874
|
+
}
|
|
875
|
+
throw annotateTensorLoadError(error, name, location, {
|
|
876
|
+
tensorLoadStage: 'materializeTensorToGPU',
|
|
877
|
+
toGPU: true,
|
|
878
|
+
streamedUpload,
|
|
879
|
+
});
|
|
880
|
+
}
|
|
803
881
|
|
|
804
882
|
for (const buffer of result.allocatedBuffers) {
|
|
805
883
|
this.gpuBuffers.add(buffer);
|
|
@@ -859,7 +937,7 @@ export class DopplerLoader {
|
|
|
859
937
|
let pendingBytes = null;
|
|
860
938
|
const writeAlignedChunk = (bytes) => {
|
|
861
939
|
if (bytes.byteLength === 0) return;
|
|
862
|
-
device.queue.writeBuffer(raw, dstOffset, bytes,
|
|
940
|
+
device.queue.writeBuffer(raw, dstOffset, bytes, 0, bytes.byteLength);
|
|
863
941
|
dstOffset += bytes.byteLength;
|
|
864
942
|
};
|
|
865
943
|
const uploadChunk = (bytes) => {
|
|
@@ -16,6 +16,7 @@ import { createTensor } from '../gpu/tensor.js';
|
|
|
16
16
|
import { castF16ToF32 } from '../gpu/kernel-selector.js';
|
|
17
17
|
import { releaseBuffer } from '../memory/buffer-pool.js';
|
|
18
18
|
import { loadTensorRange } from './tensors/tensor-reader.js';
|
|
19
|
+
import { hasSourceTransform } from './tensors/source-transform.js';
|
|
19
20
|
|
|
20
21
|
// ============================================================================
|
|
21
22
|
// Constants
|
|
@@ -61,6 +62,34 @@ function createRangeBackedWeightBuffer(ctx, name, location) {
|
|
|
61
62
|
return createCpuWeightBuffer(source, dtype, layout, location.shape, name);
|
|
62
63
|
}
|
|
63
64
|
|
|
65
|
+
function shouldUseRangeBackedEmbeddingSource(ctx, name, location) {
|
|
66
|
+
if (!location) {
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
if (String(location.dtype ?? '').toUpperCase() === 'F16' && ctx.hostHasShaderF16 === false) {
|
|
70
|
+
log.info(
|
|
71
|
+
'Loader',
|
|
72
|
+
`Embedding "${name}" range-backed: F16 source on device without shader-f16 support.`
|
|
73
|
+
);
|
|
74
|
+
return true;
|
|
75
|
+
}
|
|
76
|
+
if (hasSourceTransform(location) && typeof ctx.loadShardRange === 'function') {
|
|
77
|
+
log.info(
|
|
78
|
+
'Loader',
|
|
79
|
+
`Embedding "${name}" range-backed: sourceTransform.kind="${location.sourceTransform.kind}" defers full materialization.`
|
|
80
|
+
);
|
|
81
|
+
return true;
|
|
82
|
+
}
|
|
83
|
+
const stream = ctx.shouldStreamLargeWeight(name, location, 'Embedding');
|
|
84
|
+
if (!stream) {
|
|
85
|
+
log.info(
|
|
86
|
+
'Loader',
|
|
87
|
+
`Embedding "${name}" GPU-resident: shouldStreamLargeWeight returned false.`
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
return stream;
|
|
91
|
+
}
|
|
92
|
+
|
|
64
93
|
// ============================================================================
|
|
65
94
|
// Main Function
|
|
66
95
|
// ============================================================================
|
|
@@ -83,11 +112,7 @@ export async function loadEmbeddings(ctx) {
|
|
|
83
112
|
|
|
84
113
|
for (const name of candidates) {
|
|
85
114
|
const loc = ctx.tensorLocations.get(name);
|
|
86
|
-
const
|
|
87
|
-
&& String(loc.dtype ?? '').toUpperCase() === 'F16'
|
|
88
|
-
&& ctx.hostHasShaderF16 === false;
|
|
89
|
-
const shouldStream = requiresCpuFloatFallback
|
|
90
|
-
|| (loc ? ctx.shouldStreamLargeWeight(name, loc, 'Embedding') : false);
|
|
115
|
+
const shouldStream = shouldUseRangeBackedEmbeddingSource(ctx, name, loc);
|
|
91
116
|
|
|
92
117
|
// Load tensor (to CPU if streaming, to GPU otherwise)
|
|
93
118
|
const tensor = shouldStream
|