doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { float32ToFloat16 } from '../../converter/quantizer.js';
|
|
2
|
+
import { normalizeTensorSourceTransform } from '../../formats/rdrr/source-transform-contract.js';
|
|
2
3
|
|
|
3
4
|
function computeElementCount(shape, tensorName) {
|
|
4
5
|
if (!Array.isArray(shape)) {
|
|
@@ -211,6 +212,26 @@ function validateLiteRTStorageEncoding(storageEncoding, tensorName) {
|
|
|
211
212
|
}
|
|
212
213
|
}
|
|
213
214
|
|
|
215
|
+
function validateLiteRTStorageLaneOrder(storageLaneOrder, storageBlockSize, tensorName) {
|
|
216
|
+
if (!Array.isArray(storageLaneOrder) || storageLaneOrder.length !== storageBlockSize) {
|
|
217
|
+
throw new Error(
|
|
218
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform requires storageLaneOrder ` +
|
|
219
|
+
`with ${storageBlockSize} entries.`
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
const seen = new Set();
|
|
223
|
+
for (let index = 0; index < storageLaneOrder.length; index++) {
|
|
224
|
+
const value = Number(storageLaneOrder[index]);
|
|
225
|
+
if (!Number.isInteger(value) || value < 0 || value >= storageBlockSize || seen.has(value)) {
|
|
226
|
+
throw new Error(
|
|
227
|
+
`[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT storageLaneOrder ` +
|
|
228
|
+
`${JSON.stringify(storageLaneOrder)}.`
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
seen.add(value);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
214
235
|
function resolveLiteRTScaleValue(storedScale, transform, tensorName, rowLabel) {
|
|
215
236
|
const scaleSemantics = String(transform?.scaleSemantics || '').trim().toLowerCase();
|
|
216
237
|
if (scaleSemantics === 'step') {
|
|
@@ -245,13 +266,14 @@ function getLiteRTCompanionByteLength(companionSource, tensorName, label, expect
|
|
|
245
266
|
}
|
|
246
267
|
|
|
247
268
|
export function getSourceTransformSpec(location, tensorName) {
|
|
248
|
-
|
|
269
|
+
const transform = normalizeTensorSourceTransform(location, tensorName);
|
|
270
|
+
if (!transform) {
|
|
249
271
|
return null;
|
|
250
272
|
}
|
|
251
|
-
const transform = location.sourceTransform;
|
|
252
273
|
if (transform.kind === 'affine_dequant') {
|
|
253
274
|
const elementCount = computeElementCount(location.shape, tensorName);
|
|
254
275
|
return {
|
|
276
|
+
transform,
|
|
255
277
|
kind: 'affine_dequant',
|
|
256
278
|
elementCount,
|
|
257
279
|
outputByteLength: elementCount * 2,
|
|
@@ -314,6 +336,7 @@ export function getSourceTransformSpec(location, tensorName) {
|
|
|
314
336
|
}
|
|
315
337
|
}
|
|
316
338
|
return {
|
|
339
|
+
transform,
|
|
317
340
|
kind: 'litert_rowwise_dequant',
|
|
318
341
|
rows,
|
|
319
342
|
cols,
|
|
@@ -397,6 +420,33 @@ export function getSourceTransformSpec(location, tensorName) {
|
|
|
397
420
|
);
|
|
398
421
|
}
|
|
399
422
|
}
|
|
423
|
+
const scaleCompanionDtype = String(transform?.scaleCompanionDtype || '').toUpperCase();
|
|
424
|
+
const scaleCompanionDequant = scaleCompanionDtype === 'UINT8'
|
|
425
|
+
? transform.scaleCompanionDequant
|
|
426
|
+
: null;
|
|
427
|
+
if (scaleCompanionDtype === 'UINT8') {
|
|
428
|
+
if (!scaleCompanionDequant || typeof scaleCompanionDequant !== 'object') {
|
|
429
|
+
throw new Error(
|
|
430
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has UINT8 scale companion without scaleCompanionDequant metadata.`
|
|
431
|
+
);
|
|
432
|
+
}
|
|
433
|
+
const companionScale = Number(scaleCompanionDequant.scale);
|
|
434
|
+
const companionZeroPoint = Number(scaleCompanionDequant.zeroPoint);
|
|
435
|
+
if (!Number.isFinite(companionScale) || companionScale <= 0) {
|
|
436
|
+
throw new Error(
|
|
437
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has invalid scaleCompanionDequant.scale ${scaleCompanionDequant.scale}.`
|
|
438
|
+
);
|
|
439
|
+
}
|
|
440
|
+
if (!Number.isSafeInteger(companionZeroPoint)) {
|
|
441
|
+
throw new Error(
|
|
442
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has invalid scaleCompanionDequant.zeroPoint ${scaleCompanionDequant.zeroPoint}.`
|
|
443
|
+
);
|
|
444
|
+
}
|
|
445
|
+
} else if (scaleCompanionDtype !== '') {
|
|
446
|
+
throw new Error(
|
|
447
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has unsupported scaleCompanionDtype "${scaleCompanionDtype}".`
|
|
448
|
+
);
|
|
449
|
+
}
|
|
400
450
|
const rawStorageRowBytes = computeSourceByteLength(storageCols, transform.sourceDtype, tensorName);
|
|
401
451
|
const sourceByteLength = computeSourceByteLength(
|
|
402
452
|
storageRows * storageCols,
|
|
@@ -404,13 +454,14 @@ export function getSourceTransformSpec(location, tensorName) {
|
|
|
404
454
|
tensorName
|
|
405
455
|
);
|
|
406
456
|
return {
|
|
457
|
+
transform,
|
|
407
458
|
kind: 'litert_axis_dequant',
|
|
408
459
|
rows,
|
|
409
460
|
cols,
|
|
410
461
|
storageRows,
|
|
411
462
|
storageCols,
|
|
412
463
|
rawStorageRowBytes,
|
|
413
|
-
scaleRowBytes: 4,
|
|
464
|
+
scaleRowBytes: scaleCompanionDtype === 'UINT8' ? 4 : 4,
|
|
414
465
|
targetRowBytes: cols * 2,
|
|
415
466
|
outputByteLength: rows * cols * 2,
|
|
416
467
|
sourceByteLength,
|
|
@@ -419,6 +470,96 @@ export function getSourceTransformSpec(location, tensorName) {
|
|
|
419
470
|
};
|
|
420
471
|
}
|
|
421
472
|
|
|
473
|
+
if (transform.kind === 'litert_axis_blocked_dequant') {
|
|
474
|
+
if (transform.scheme !== 'per_axis_affine') {
|
|
475
|
+
throw new Error(
|
|
476
|
+
`[DopplerLoader] Tensor "${tensorName}" has unsupported sourceTransform.scheme "${transform.scheme}".`
|
|
477
|
+
);
|
|
478
|
+
}
|
|
479
|
+
validateLiteRTTransformTarget(location, tensorName, transform, 'LiteRT blocked axis sourceTransform');
|
|
480
|
+
validateLiteRTStorageEncoding(transform.storageEncoding, tensorName);
|
|
481
|
+
if (!Array.isArray(transform.storageShape) || transform.storageShape.length !== 2) {
|
|
482
|
+
throw new Error(
|
|
483
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform requires storageShape=[rows, cols].`
|
|
484
|
+
);
|
|
485
|
+
}
|
|
486
|
+
const storageRows = Number(transform.storageShape[0]);
|
|
487
|
+
const storageCols = Number(transform.storageShape[1]);
|
|
488
|
+
const storageBlockSize = Number(transform.storageBlockSize);
|
|
489
|
+
if (
|
|
490
|
+
!Number.isInteger(storageRows)
|
|
491
|
+
|| storageRows <= 0
|
|
492
|
+
|| !Number.isInteger(storageCols)
|
|
493
|
+
|| storageCols <= 0
|
|
494
|
+
|| !Number.isInteger(storageBlockSize)
|
|
495
|
+
|| storageBlockSize <= 0
|
|
496
|
+
) {
|
|
497
|
+
throw new Error(
|
|
498
|
+
`[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT blocked storage shape ` +
|
|
499
|
+
`${JSON.stringify({ storageShape: transform.storageShape, storageBlockSize: transform.storageBlockSize })}.`
|
|
500
|
+
);
|
|
501
|
+
}
|
|
502
|
+
if (transform.quantAxis !== 0) {
|
|
503
|
+
throw new Error(
|
|
504
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform only supports quantAxis=0.`
|
|
505
|
+
);
|
|
506
|
+
}
|
|
507
|
+
validateLiteRTStorageLaneOrder(transform.storageLaneOrder, storageBlockSize, tensorName);
|
|
508
|
+
if (storageCols !== rows || storageRows * storageBlockSize !== cols) {
|
|
509
|
+
throw new Error(
|
|
510
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform expects storageShape ` +
|
|
511
|
+
`[${cols / storageBlockSize}, ${rows}] for blockSize=${storageBlockSize}, got [${storageRows}, ${storageCols}].`
|
|
512
|
+
);
|
|
513
|
+
}
|
|
514
|
+
const scaleSource = transform.scaleSource;
|
|
515
|
+
if (!scaleSource || typeof scaleSource !== 'object') {
|
|
516
|
+
throw new Error(
|
|
517
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform is missing scaleSource.`
|
|
518
|
+
);
|
|
519
|
+
}
|
|
520
|
+
getLiteRTCompanionByteLength(scaleSource, tensorName, 'blocked axis scale', rows * 4);
|
|
521
|
+
if (transform.sumSource != null && typeof transform.sumSource !== 'object') {
|
|
522
|
+
throw new Error(
|
|
523
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform has invalid sumSource.`
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
if (transform.sumSource != null) {
|
|
527
|
+
getLiteRTCompanionByteLength(transform.sumSource, tensorName, 'blocked axis sum', rows * 4);
|
|
528
|
+
}
|
|
529
|
+
if (transform.scaleSemantics !== 'step' && transform.scaleSemantics !== 'qmax_abs') {
|
|
530
|
+
throw new Error(
|
|
531
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform has unsupported scaleSemantics "${transform.scaleSemantics}".`
|
|
532
|
+
);
|
|
533
|
+
}
|
|
534
|
+
if (transform.scaleSemantics === 'qmax_abs') {
|
|
535
|
+
const scaleDivisor = Number(transform.scaleDivisor);
|
|
536
|
+
if (!Number.isFinite(scaleDivisor) || scaleDivisor <= 0) {
|
|
537
|
+
throw new Error(
|
|
538
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform requires scaleDivisor > 0 for scaleSemantics="qmax_abs".`
|
|
539
|
+
);
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
const storageElementBytes = computeSourceByteLength(storageBlockSize, transform.sourceDtype, tensorName);
|
|
543
|
+
const rawStorageRowBytes = storageCols * storageElementBytes;
|
|
544
|
+
return {
|
|
545
|
+
transform,
|
|
546
|
+
kind: 'litert_axis_blocked_dequant',
|
|
547
|
+
rows,
|
|
548
|
+
cols,
|
|
549
|
+
storageRows,
|
|
550
|
+
storageCols,
|
|
551
|
+
rawStorageRowBytes,
|
|
552
|
+
storageElementBytes,
|
|
553
|
+
scaleRowBytes: 4,
|
|
554
|
+
targetRowBytes: cols * 2,
|
|
555
|
+
outputByteLength: rows * cols * 2,
|
|
556
|
+
sourceByteLength: storageRows * rawStorageRowBytes,
|
|
557
|
+
quantAxis: transform.quantAxis,
|
|
558
|
+
storageBlockSize,
|
|
559
|
+
storageLaneOrder: transform.storageLaneOrder.map((value) => Number(value)),
|
|
560
|
+
};
|
|
561
|
+
}
|
|
562
|
+
|
|
422
563
|
throw new Error(
|
|
423
564
|
`[DopplerLoader] Tensor "${tensorName}" has unsupported sourceTransform.kind "${transform.kind}".`
|
|
424
565
|
);
|
|
@@ -438,9 +579,9 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
438
579
|
return rawBytes;
|
|
439
580
|
}
|
|
440
581
|
|
|
441
|
-
const
|
|
582
|
+
const spec = getSourceTransformSpec(location, tensorName);
|
|
583
|
+
const transform = spec.transform;
|
|
442
584
|
if (transform.kind === 'litert_rowwise_dequant') {
|
|
443
|
-
const spec = getSourceTransformSpec(location, tensorName);
|
|
444
585
|
const scaleBytes = options.scaleBytes;
|
|
445
586
|
if (!(scaleBytes instanceof Uint8Array)) {
|
|
446
587
|
throw new Error(
|
|
@@ -549,7 +690,6 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
549
690
|
}
|
|
550
691
|
|
|
551
692
|
if (transform.kind === 'litert_axis_dequant') {
|
|
552
|
-
const spec = getSourceTransformSpec(location, tensorName);
|
|
553
693
|
const scaleBytes = options.scaleBytes;
|
|
554
694
|
if (!(scaleBytes instanceof Uint8Array)) {
|
|
555
695
|
throw new Error(
|
|
@@ -598,6 +738,51 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
598
738
|
? new DataView(sumBytes.buffer, sumBytes.byteOffset, sumBytes.byteLength)
|
|
599
739
|
: null;
|
|
600
740
|
const out = new Uint16Array(rowCount * spec.cols);
|
|
741
|
+
const scaleCompanionDtype = String(transform.scaleCompanionDtype || '').toUpperCase();
|
|
742
|
+
const scaleCompanion = scaleCompanionDtype === 'UINT8'
|
|
743
|
+
? {
|
|
744
|
+
scale: Number(transform.scaleCompanionDequant?.scale),
|
|
745
|
+
zeroPoint: Number(transform.scaleCompanionDequant?.zeroPoint),
|
|
746
|
+
}
|
|
747
|
+
: null;
|
|
748
|
+
const scaleSubCount = scaleCompanion ? spec.scaleRowBytes : 1;
|
|
749
|
+
const scaleSubBlockWidth = scaleCompanion
|
|
750
|
+
? Math.floor(spec.cols / scaleSubCount)
|
|
751
|
+
: spec.cols;
|
|
752
|
+
if (!Number.isInteger(scaleSubCount) || scaleSubCount <= 0) {
|
|
753
|
+
throw new Error(
|
|
754
|
+
`[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT axis uint8 companion block count ${scaleSubCount}.`
|
|
755
|
+
);
|
|
756
|
+
}
|
|
757
|
+
if (!Number.isInteger(scaleSubBlockWidth) || scaleSubBlockWidth <= 0) {
|
|
758
|
+
throw new Error(
|
|
759
|
+
`[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT axis uint8 companion layout for width ${spec.cols} and sub-count ${scaleSubCount}.`
|
|
760
|
+
);
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
// scaleBytes was loaded for the `[rowStart, rowStart + rowCount)` window,
|
|
764
|
+
// so its byte offsets are relative to `rowStart`. Subtract `rowStart`
|
|
765
|
+
// before indexing into scaleView; keep the caller contract of passing
|
|
766
|
+
// absolute row indices so error messages still name the real row.
|
|
767
|
+
const resolveScale = (logicalRow, col = 0) => {
|
|
768
|
+
const localRow = logicalRow - rowStart;
|
|
769
|
+
const rowScaleOffset = localRow * spec.scaleRowBytes;
|
|
770
|
+
const storedScale = scaleCompanion
|
|
771
|
+
? (scaleView.getUint8(rowScaleOffset + Math.min(Math.floor(col / scaleSubBlockWidth), scaleSubCount - 1))
|
|
772
|
+
- scaleCompanion.zeroPoint) * scaleCompanion.scale
|
|
773
|
+
: scaleView.getFloat32(rowScaleOffset, true);
|
|
774
|
+
if (!Number.isFinite(storedScale) || storedScale <= 0) {
|
|
775
|
+
throw new Error(
|
|
776
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT axis row ${logicalRow} has invalid scale ${storedScale}.`
|
|
777
|
+
);
|
|
778
|
+
}
|
|
779
|
+
return resolveLiteRTScaleValue(
|
|
780
|
+
storedScale,
|
|
781
|
+
transform,
|
|
782
|
+
tensorName,
|
|
783
|
+
`LiteRT axis row ${logicalRow}`
|
|
784
|
+
);
|
|
785
|
+
};
|
|
601
786
|
|
|
602
787
|
if (spec.quantAxis === 1) {
|
|
603
788
|
const expectedRawBytes = rowCount * spec.rawStorageRowBytes;
|
|
@@ -608,18 +793,6 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
608
793
|
);
|
|
609
794
|
}
|
|
610
795
|
for (let row = 0; row < rowCount; row++) {
|
|
611
|
-
const scale = scaleView.getFloat32(row * spec.scaleRowBytes, true);
|
|
612
|
-
if (!Number.isFinite(scale) || scale <= 0) {
|
|
613
|
-
throw new Error(
|
|
614
|
-
`[DopplerLoader] Tensor "${tensorName}" LiteRT axis row ${rowStart + row} has invalid scale ${scale}.`
|
|
615
|
-
);
|
|
616
|
-
}
|
|
617
|
-
const resolvedScale = resolveLiteRTScaleValue(
|
|
618
|
-
scale,
|
|
619
|
-
transform,
|
|
620
|
-
tensorName,
|
|
621
|
-
`LiteRT axis row ${rowStart + row}`
|
|
622
|
-
);
|
|
623
796
|
const rawRowOffset = row * spec.rawStorageRowBytes;
|
|
624
797
|
const rowBytes = rawBytes.subarray(rawRowOffset, rawRowOffset + spec.rawStorageRowBytes);
|
|
625
798
|
const storedRowSum = hasSumSource
|
|
@@ -641,6 +814,7 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
641
814
|
}
|
|
642
815
|
const outRowOffset = row * spec.cols;
|
|
643
816
|
for (let col = 0; col < spec.cols; col++) {
|
|
817
|
+
const scaleForCol = resolveScale(rowStart + row, col);
|
|
644
818
|
const quantized = rowZeroPoint == null
|
|
645
819
|
? readQuantizedValue(
|
|
646
820
|
rowBytes,
|
|
@@ -656,7 +830,7 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
656
830
|
transform.storageEncoding
|
|
657
831
|
) - rowZeroPoint
|
|
658
832
|
);
|
|
659
|
-
out[outRowOffset + col] = float32ToFloat16(quantized *
|
|
833
|
+
out[outRowOffset + col] = float32ToFloat16(quantized * scaleForCol);
|
|
660
834
|
}
|
|
661
835
|
}
|
|
662
836
|
return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
|
|
@@ -697,19 +871,6 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
697
871
|
for (let row = 0; row < rowCount; row++) {
|
|
698
872
|
const logicalRow = rowStart + row;
|
|
699
873
|
const localColumnIndex = localColumnBase + row;
|
|
700
|
-
const scale = scaleView.getFloat32(row * spec.scaleRowBytes, true);
|
|
701
|
-
if (!Number.isFinite(scale) || scale <= 0) {
|
|
702
|
-
throw new Error(
|
|
703
|
-
`[DopplerLoader] Tensor "${tensorName}" LiteRT axis row ${logicalRow} has invalid scale ${scale}.`
|
|
704
|
-
);
|
|
705
|
-
}
|
|
706
|
-
const resolvedScale = resolveLiteRTScaleValue(
|
|
707
|
-
scale,
|
|
708
|
-
transform,
|
|
709
|
-
tensorName,
|
|
710
|
-
`LiteRT axis row ${logicalRow}`
|
|
711
|
-
);
|
|
712
|
-
|
|
713
874
|
const storedValues = new Array(spec.cols);
|
|
714
875
|
let storedSum = 0;
|
|
715
876
|
for (let col = 0; col < spec.cols; col++) {
|
|
@@ -737,6 +898,7 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
737
898
|
|
|
738
899
|
const outRowOffset = row * spec.cols;
|
|
739
900
|
for (let col = 0; col < spec.cols; col++) {
|
|
901
|
+
const scaleForCol = resolveScale(logicalRow, col);
|
|
740
902
|
const quantized = rowZeroPoint == null
|
|
741
903
|
? readQuantizedValue(
|
|
742
904
|
rowSlices[col],
|
|
@@ -745,14 +907,135 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
745
907
|
transform.storageEncoding
|
|
746
908
|
)
|
|
747
909
|
: (storedValues[col] - rowZeroPoint);
|
|
748
|
-
out[outRowOffset + col] = float32ToFloat16(quantized *
|
|
910
|
+
out[outRowOffset + col] = float32ToFloat16(quantized * scaleForCol);
|
|
749
911
|
}
|
|
750
912
|
}
|
|
751
913
|
return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
|
|
752
914
|
}
|
|
753
915
|
|
|
754
|
-
|
|
916
|
+
if (transform.kind === 'litert_axis_blocked_dequant') {
|
|
917
|
+
const scaleBytes = options.scaleBytes;
|
|
918
|
+
if (!(scaleBytes instanceof Uint8Array)) {
|
|
919
|
+
throw new Error(
|
|
920
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis sourceTransform requires scaleBytes.`
|
|
921
|
+
);
|
|
922
|
+
}
|
|
923
|
+
const sumBytes = options.sumBytes ?? null;
|
|
924
|
+
const hasSumSource = transform.sumSource != null;
|
|
925
|
+
if (hasSumSource && !(sumBytes instanceof Uint8Array)) {
|
|
926
|
+
throw new Error(
|
|
927
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis sourceTransform requires sumBytes.`
|
|
928
|
+
);
|
|
929
|
+
}
|
|
930
|
+
const rowStart = options.rowStart == null ? 0 : Number(options.rowStart);
|
|
931
|
+
const rowCount = options.rowCount == null ? spec.rows : Number(options.rowCount);
|
|
932
|
+
if (!Number.isInteger(rowStart) || rowStart < 0) {
|
|
933
|
+
throw new Error(
|
|
934
|
+
`[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT blocked axis rowStart ${options.rowStart}.`
|
|
935
|
+
);
|
|
936
|
+
}
|
|
937
|
+
if (!Number.isInteger(rowCount) || rowCount <= 0) {
|
|
938
|
+
throw new Error(
|
|
939
|
+
`[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT blocked axis rowCount ${options.rowCount}.`
|
|
940
|
+
);
|
|
941
|
+
}
|
|
942
|
+
if (rowStart + rowCount > spec.rows) {
|
|
943
|
+
throw new Error(
|
|
944
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis row range (${rowStart}..${rowStart + rowCount}) exceeds ${spec.rows} rows.`
|
|
945
|
+
);
|
|
946
|
+
}
|
|
947
|
+
if (scaleBytes.byteLength !== rowCount * spec.scaleRowBytes) {
|
|
948
|
+
throw new Error(
|
|
949
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis scale byte size mismatch. ` +
|
|
950
|
+
`Expected ${rowCount * spec.scaleRowBytes}, got ${scaleBytes.byteLength}.`
|
|
951
|
+
);
|
|
952
|
+
}
|
|
953
|
+
if (hasSumSource && sumBytes.byteLength !== rowCount * 4) {
|
|
954
|
+
throw new Error(
|
|
955
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis sum byte size mismatch. ` +
|
|
956
|
+
`Expected ${rowCount * 4}, got ${sumBytes.byteLength}.`
|
|
957
|
+
);
|
|
958
|
+
}
|
|
959
|
+
const expectedRawBytes = spec.storageRows * rowCount * spec.storageElementBytes;
|
|
960
|
+
if (rawBytes.byteLength !== expectedRawBytes) {
|
|
961
|
+
throw new Error(
|
|
962
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis raw byte size mismatch. ` +
|
|
963
|
+
`Expected ${expectedRawBytes}, got ${rawBytes.byteLength}.`
|
|
964
|
+
);
|
|
965
|
+
}
|
|
966
|
+
const scaleView = new DataView(scaleBytes.buffer, scaleBytes.byteOffset, scaleBytes.byteLength);
|
|
967
|
+
const sumView = hasSumSource
|
|
968
|
+
? new DataView(sumBytes.buffer, sumBytes.byteOffset, sumBytes.byteLength)
|
|
969
|
+
: null;
|
|
970
|
+
const out = new Uint16Array(rowCount * spec.cols);
|
|
971
|
+
for (let row = 0; row < rowCount; row++) {
|
|
972
|
+
const logicalRow = rowStart + row;
|
|
973
|
+
const scale = scaleView.getFloat32(row * spec.scaleRowBytes, true);
|
|
974
|
+
if (!Number.isFinite(scale) || scale <= 0) {
|
|
975
|
+
throw new Error(
|
|
976
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis row ${logicalRow} has invalid scale ${scale}.`
|
|
977
|
+
);
|
|
978
|
+
}
|
|
979
|
+
const resolvedScale = resolveLiteRTScaleValue(
|
|
980
|
+
scale,
|
|
981
|
+
transform,
|
|
982
|
+
tensorName,
|
|
983
|
+
`LiteRT blocked axis row ${logicalRow}`
|
|
984
|
+
);
|
|
985
|
+
const storedRowSum = hasSumSource
|
|
986
|
+
? sumView.getInt32(row * 4, true)
|
|
987
|
+
: null;
|
|
988
|
+
let storedSum = 0;
|
|
989
|
+
if (storedRowSum != null) {
|
|
990
|
+
for (let storageRow = 0; storageRow < spec.storageRows; storageRow++) {
|
|
991
|
+
const baseOffset = storageRow * rowCount * spec.storageElementBytes + row * spec.storageElementBytes;
|
|
992
|
+
const blockBytes = rawBytes.subarray(baseOffset, baseOffset + spec.storageElementBytes);
|
|
993
|
+
for (let lane = 0; lane < spec.storageBlockSize; lane++) {
|
|
994
|
+
storedSum += readStoredQuantizedValue(
|
|
995
|
+
blockBytes,
|
|
996
|
+
lane,
|
|
997
|
+
transform.sourceDtype,
|
|
998
|
+
transform.storageEncoding
|
|
999
|
+
);
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
const rowZeroPoint = storedRowSum == null
|
|
1004
|
+
? null
|
|
1005
|
+
: (storedSum - storedRowSum) / spec.cols;
|
|
1006
|
+
if (rowZeroPoint != null && !Number.isFinite(rowZeroPoint)) {
|
|
1007
|
+
throw new Error(
|
|
1008
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis row ${logicalRow} has invalid derived rowZeroPoint ${rowZeroPoint}.`
|
|
1009
|
+
);
|
|
1010
|
+
}
|
|
1011
|
+
const outRowOffset = row * spec.cols;
|
|
1012
|
+
for (let storageRow = 0; storageRow < spec.storageRows; storageRow++) {
|
|
1013
|
+
const baseOffset = storageRow * rowCount * spec.storageElementBytes + row * spec.storageElementBytes;
|
|
1014
|
+
const blockBytes = rawBytes.subarray(baseOffset, baseOffset + spec.storageElementBytes);
|
|
1015
|
+
for (let lane = 0; lane < spec.storageBlockSize; lane++) {
|
|
1016
|
+
const stored = readStoredQuantizedValue(
|
|
1017
|
+
blockBytes,
|
|
1018
|
+
lane,
|
|
1019
|
+
transform.sourceDtype,
|
|
1020
|
+
transform.storageEncoding
|
|
1021
|
+
);
|
|
1022
|
+
const quantized = rowZeroPoint == null
|
|
1023
|
+
? readQuantizedValue(
|
|
1024
|
+
blockBytes,
|
|
1025
|
+
lane,
|
|
1026
|
+
transform.sourceDtype,
|
|
1027
|
+
transform.storageEncoding
|
|
1028
|
+
)
|
|
1029
|
+
: (stored - rowZeroPoint);
|
|
1030
|
+
const logicalCol = storageRow * spec.storageBlockSize + spec.storageLaneOrder[lane];
|
|
1031
|
+
out[outRowOffset + logicalCol] = float32ToFloat16(quantized * resolvedScale);
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
|
|
1036
|
+
}
|
|
755
1037
|
|
|
1038
|
+
const affineTransform = transform;
|
|
756
1039
|
const elementCount = computeElementCount(location.shape, tensorName);
|
|
757
1040
|
const expectedBytes = computeSourceByteLength(
|
|
758
1041
|
elementCount,
|
|
@@ -19,6 +19,8 @@ export interface TensorLoadConfig {
|
|
|
19
19
|
useFusedQ4K: boolean;
|
|
20
20
|
/** Q4K weight materialization mode derived from the resolved execution graph */
|
|
21
21
|
q4kMaterializationMode?: 'dense' | 'fused' | 'mixed';
|
|
22
|
+
/** Tensor roles that require raw Q4K buffers because their graph step pins a fused Q4K kernel */
|
|
23
|
+
q4kFusedRoles?: string[];
|
|
22
24
|
/** Debug controls for Q4K loading/dequantization */
|
|
23
25
|
loaderDebug?: LoaderDebugConfigSchema | null;
|
|
24
26
|
/** Keep weights as F32 (disable F16 downcasting) */
|
|
@@ -99,15 +101,6 @@ export declare function loadQ4KDequant(
|
|
|
99
101
|
config: TensorLoadConfig
|
|
100
102
|
): Promise<TensorLoadResult>;
|
|
101
103
|
|
|
102
|
-
/**
|
|
103
|
-
* Load Q4K tensor to GPU with both dense and fused-compatible materializations.
|
|
104
|
-
*/
|
|
105
|
-
export declare function loadQ4KMixed(
|
|
106
|
-
shardData: Uint8Array,
|
|
107
|
-
location: TensorLocation,
|
|
108
|
-
name: string,
|
|
109
|
-
config: TensorLoadConfig
|
|
110
|
-
): Promise<TensorLoadResult>;
|
|
111
104
|
|
|
112
105
|
/**
|
|
113
106
|
* Load Q6K tensor to GPU.
|
|
@@ -17,6 +17,7 @@ import { hasSourceTransform } from './source-transform.js';
|
|
|
17
17
|
// ============================================================================
|
|
18
18
|
|
|
19
19
|
let loggedF32UpcastNonMatmul = false;
|
|
20
|
+
let loggedQ4KLimitFallback = false;
|
|
20
21
|
|
|
21
22
|
function isGpuBufferInstance(value) {
|
|
22
23
|
return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
|
|
@@ -128,7 +129,10 @@ function isEmbeddingRole(location) {
|
|
|
128
129
|
|
|
129
130
|
export function shouldUseFusedQ4K(location, config) {
|
|
130
131
|
if (!config.useFusedQ4K) return false;
|
|
132
|
+
return canUseFusedQ4KStorage(location, config);
|
|
133
|
+
}
|
|
131
134
|
|
|
135
|
+
function canUseFusedQ4KStorage(location, config) {
|
|
132
136
|
const caps = config.gpuCapabilities || getKernelCapabilities();
|
|
133
137
|
if (!caps?.hasSubgroups) return false;
|
|
134
138
|
|
|
@@ -141,6 +145,75 @@ export function shouldUseFusedQ4K(location, config) {
|
|
|
141
145
|
return true;
|
|
142
146
|
}
|
|
143
147
|
|
|
148
|
+
function getQ4KDenseMaterializedSizeBytes(location, config) {
|
|
149
|
+
if (!Array.isArray(location.shape) || location.shape.length === 0) {
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
const elementCount = getShapeElementCount(location.shape);
|
|
153
|
+
if (!Number.isFinite(elementCount) || elementCount <= 0) {
|
|
154
|
+
return null;
|
|
155
|
+
}
|
|
156
|
+
const outputDtype = getQ4KOutputDtype(location, config);
|
|
157
|
+
const bytesPerElement = outputDtype === 'f16' ? 2 : 4;
|
|
158
|
+
return elementCount * bytesPerElement;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function getMaxStorageBufferBindingSize() {
|
|
162
|
+
const device = getDevice();
|
|
163
|
+
const maxStorage = device?.limits?.maxStorageBufferBindingSize;
|
|
164
|
+
return Number.isFinite(maxStorage) && maxStorage > 0 ? maxStorage : null;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function resolveQ4KLimitFallback(location, config) {
|
|
168
|
+
if (location?.dtype !== 'Q4_K_M' && location?.dtype !== 'Q4_K') {
|
|
169
|
+
return {
|
|
170
|
+
denseExceedsBindingLimit: false,
|
|
171
|
+
limitFallbackEligible: false,
|
|
172
|
+
denseSizeBytes: null,
|
|
173
|
+
maxBindingSizeBytes: null,
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const denseSizeBytes = getQ4KDenseMaterializedSizeBytes(location, config);
|
|
178
|
+
const maxBindingSizeBytes = getMaxStorageBufferBindingSize();
|
|
179
|
+
const denseExceedsBindingLimit = (
|
|
180
|
+
denseSizeBytes != null
|
|
181
|
+
&& maxBindingSizeBytes != null
|
|
182
|
+
&& denseSizeBytes > maxBindingSizeBytes
|
|
183
|
+
);
|
|
184
|
+
const packedSizeBytes = Number.isFinite(location.size) ? location.size : null;
|
|
185
|
+
const packedFitsBindingLimit = (
|
|
186
|
+
packedSizeBytes != null
|
|
187
|
+
&& maxBindingSizeBytes != null
|
|
188
|
+
&& packedSizeBytes <= maxBindingSizeBytes
|
|
189
|
+
);
|
|
190
|
+
const limitFallbackEligible = (
|
|
191
|
+
denseExceedsBindingLimit
|
|
192
|
+
&& config.keepF32Weights !== true
|
|
193
|
+
&& packedFitsBindingLimit
|
|
194
|
+
&& canUseFusedQ4KStorage(location, config)
|
|
195
|
+
);
|
|
196
|
+
|
|
197
|
+
return {
|
|
198
|
+
denseExceedsBindingLimit,
|
|
199
|
+
limitFallbackEligible,
|
|
200
|
+
denseSizeBytes,
|
|
201
|
+
maxBindingSizeBytes,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function logQ4KLimitFallbackOnce(name, fallback) {
|
|
206
|
+
if (loggedQ4KLimitFallback) {
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
loggedQ4KLimitFallback = true;
|
|
210
|
+
log.warn(
|
|
211
|
+
'Loader',
|
|
212
|
+
`Q4K dense materialization for "${name}" would require ${fallback.denseSizeBytes} bytes, ` +
|
|
213
|
+
`exceeding maxStorageBufferBindingSize=${fallback.maxBindingSizeBytes}; retaining packed Q4K for fused matmul.`
|
|
214
|
+
);
|
|
215
|
+
}
|
|
216
|
+
|
|
144
217
|
// ============================================================================
|
|
145
218
|
// Dtype Output Selection
|
|
146
219
|
// ============================================================================
|
|
@@ -389,7 +462,7 @@ export async function loadQ4KDequant(shardData, location, name, config) {
|
|
|
389
462
|
}
|
|
390
463
|
}
|
|
391
464
|
|
|
392
|
-
|
|
465
|
+
async function loadQ4KMixed(shardData, location, name, config) {
|
|
393
466
|
const canMaterializeMixed = shouldUseFusedQ4K(location, config)
|
|
394
467
|
&& config.q4kLayout === 'row';
|
|
395
468
|
if (!canMaterializeMixed) {
|
|
@@ -693,22 +766,32 @@ const GPU_LOADER_DISPATCH = {
|
|
|
693
766
|
export async function loadTensorToGPU(shardData, location, name, config) {
|
|
694
767
|
const dtype = location.dtype;
|
|
695
768
|
const useFusedQ4K = shouldUseFusedQ4K(location, config);
|
|
769
|
+
const requiresFusedQ4KRole = Array.isArray(config?.q4kFusedRoles)
|
|
770
|
+
&& config.q4kFusedRoles.includes(location.role);
|
|
696
771
|
const caps = config?.gpuCapabilities || getKernelCapabilities();
|
|
697
772
|
const platformId = getPlatformConfig()?.platform?.id ?? null;
|
|
698
773
|
const q4kReferenceContext = getQ4KCpuReferenceContext(shardData, location, config);
|
|
699
774
|
const q4kBasicBackendClass = platformId === 'basic'
|
|
700
775
|
|| (caps?.hasSubgroups !== true && caps?.hasF16 !== true);
|
|
776
|
+
const q4kLimitFallback = resolveQ4KLimitFallback(location, config);
|
|
701
777
|
const loaderPath = selectRuleValue('loader', 'tensorLoader', 'gpuLoaderPath', {
|
|
702
778
|
dtype,
|
|
779
|
+
role: location.role ?? null,
|
|
703
780
|
useFusedQ4K,
|
|
781
|
+
requiresFusedQ4KRole,
|
|
704
782
|
q4kMaterializationMode: config.q4kMaterializationMode ?? 'dense',
|
|
705
783
|
q4kCpuReferenceEligible: q4kReferenceContext.eligible,
|
|
706
784
|
q4kBasicBackendClass,
|
|
785
|
+
q4kDenseExceedsBindingLimit: q4kLimitFallback.denseExceedsBindingLimit,
|
|
786
|
+
q4kLimitFallbackEligible: q4kLimitFallback.limitFallbackEligible,
|
|
707
787
|
});
|
|
708
788
|
const loader = GPU_LOADER_DISPATCH[loaderPath];
|
|
709
789
|
if (!loader) {
|
|
710
790
|
throw new Error(`Unknown GPU loader path: "${loaderPath}" for dtype "${dtype}"`);
|
|
711
791
|
}
|
|
792
|
+
if (loaderPath === 'q4k_fused' && q4kLimitFallback.limitFallbackEligible) {
|
|
793
|
+
logQ4KLimitFallbackOnce(name, q4kLimitFallback);
|
|
794
|
+
}
|
|
712
795
|
return loader(shardData, location, name, config);
|
|
713
796
|
}
|
|
714
797
|
|
|
@@ -161,7 +161,7 @@ function getSourceTransformScaleLocation(location, name) {
|
|
|
161
161
|
|
|
162
162
|
function getSourceTransformSumLocation(location) {
|
|
163
163
|
const transform = location?.sourceTransform;
|
|
164
|
-
const sumLocation = transform?.kind === 'litert_axis_dequant'
|
|
164
|
+
const sumLocation = transform?.kind === 'litert_axis_dequant' || transform?.kind === 'litert_axis_blocked_dequant'
|
|
165
165
|
? transform?.sumSource
|
|
166
166
|
: transform?.rowSumSource;
|
|
167
167
|
if (!sumLocation || typeof sumLocation !== 'object') {
|
|
@@ -177,6 +177,7 @@ async function materializeLocationBytes(rawBytes, location, name, loadShard, loa
|
|
|
177
177
|
if (
|
|
178
178
|
location.sourceTransform.kind === 'litert_rowwise_dequant'
|
|
179
179
|
|| location.sourceTransform.kind === 'litert_axis_dequant'
|
|
180
|
+
|| location.sourceTransform.kind === 'litert_axis_blocked_dequant'
|
|
180
181
|
) {
|
|
181
182
|
const scaleLocation = getSourceTransformScaleLocation(location, name);
|
|
182
183
|
const scaleBytes = options.scaleBytes instanceof Uint8Array
|
|
@@ -284,7 +285,11 @@ export async function loadTensorRange(location, name, byteOffset, byteLength, lo
|
|
|
284
285
|
if (hasSourceTransform(location)) {
|
|
285
286
|
const spec = getSourceTransformSpec(location, name);
|
|
286
287
|
if (
|
|
287
|
-
(
|
|
288
|
+
(
|
|
289
|
+
spec?.kind === 'litert_rowwise_dequant'
|
|
290
|
+
|| spec?.kind === 'litert_axis_dequant'
|
|
291
|
+
|| spec?.kind === 'litert_axis_blocked_dequant'
|
|
292
|
+
)
|
|
288
293
|
&& (byteOffset % spec.targetRowBytes) === 0
|
|
289
294
|
&& (byteLength % spec.targetRowBytes) === 0
|
|
290
295
|
) {
|
|
@@ -343,6 +348,27 @@ export async function loadTensorRange(location, name, byteOffset, byteLength, lo
|
|
|
343
348
|
});
|
|
344
349
|
}
|
|
345
350
|
|
|
351
|
+
if (spec.kind === 'litert_axis_blocked_dequant') {
|
|
352
|
+
const raw = new Uint8Array(spec.storageRows * rowCount * spec.storageElementBytes);
|
|
353
|
+
for (let storageRow = 0; storageRow < spec.storageRows; storageRow++) {
|
|
354
|
+
const rowBytes = await loadLocationRange(
|
|
355
|
+
location,
|
|
356
|
+
name,
|
|
357
|
+
`LiteRT blocked axis raw storage row ${storageRow}`,
|
|
358
|
+
storageRow * spec.rawStorageRowBytes + rowStart * spec.storageElementBytes,
|
|
359
|
+
rowCount * spec.storageElementBytes,
|
|
360
|
+
loadShardRange
|
|
361
|
+
);
|
|
362
|
+
raw.set(rowBytes, storageRow * rowCount * spec.storageElementBytes);
|
|
363
|
+
}
|
|
364
|
+
return materializeTensorSourceTransform(raw, location, name, {
|
|
365
|
+
scaleBytes,
|
|
366
|
+
sumBytes,
|
|
367
|
+
rowStart,
|
|
368
|
+
rowCount,
|
|
369
|
+
});
|
|
370
|
+
}
|
|
371
|
+
|
|
346
372
|
const storageColumnStart = Math.floor(rowStart / spec.storageValuesPerByte) * spec.storageValuesPerByte;
|
|
347
373
|
const storageColumnEnd = Math.ceil((rowStart + rowCount) / spec.storageValuesPerByte) * spec.storageValuesPerByte;
|
|
348
374
|
const rawSliceRowBytes = (storageColumnEnd - storageColumnStart) / spec.storageValuesPerByte;
|