doppler-gpu 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +210 -0
- package/README.md +63 -38
- package/assets/doppler.svg +57 -1601
- package/package.json +111 -32
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +486 -0
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +885 -744
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +469 -43
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +66 -10
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +69 -4
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +34 -50
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +35 -2
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +148 -3
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +8 -60
- package/src/client/wrap-pipeline-handle.d.ts +82 -0
- package/src/client/wrap-pipeline-handle.js +259 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +28 -157
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +163 -126
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +3593 -1146
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +3 -1
- package/src/config/merge-helpers.js +2 -1
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +65 -7
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +20 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +16 -0
- package/src/config/schema/converter.schema.js +11 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +13 -2
- package/src/config/schema/doppler.schema.js +140 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +39 -1
- package/src/config/schema/execution-v1.schema.js +35 -20
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +14 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +175 -2
- package/src/config/schema/manifest.schema.js +58 -3
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -9
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +183 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
- package/src/config/transforms/execution-graph-transforms.js +829 -168
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +31 -14
- package/src/converter/core.d.ts +65 -0
- package/src/converter/core.js +1153 -22
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +218 -5
- package/src/converter/quantizer.d.ts +24 -4
- package/src/converter/quantizer.js +92 -2
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +296 -38
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1733 -58
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +5 -14
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +192 -12
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +111 -2
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +10 -0
- package/src/formats/rdrr/index.js +10 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +26 -6
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +193 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +111 -2
- package/src/formats/rdrr/validation.js +260 -1
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +45 -23
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +130 -14
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +1 -8
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +458 -16
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +13 -120
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +84 -8
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +25 -22
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +77 -5
- package/src/gpu/kernels/index.js +55 -4
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +44 -2
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +172 -10
- package/src/gpu/kernels/matmul.d.ts +1 -29
- package/src/gpu/kernels/matmul.js +240 -38
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +1 -56
- package/src/gpu/kernels/moe.js +46 -33
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -12
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +6 -4
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +30 -7
- package/src/gpu/kernels/rmsnorm.js +49 -8
- package/src/gpu/kernels/rmsnorm.wgsl +15 -12
- package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +28 -21
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +8 -223
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +43 -0
- package/src/gpu/kernels/utils.js +81 -11
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +10 -12
- package/src/gpu/uniform-cache.js +25 -30
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
- package/src/inference/browser-harness-suite-helpers.js +525 -3
- package/src/inference/browser-harness-text-helpers.d.ts +89 -0
- package/src/inference/browser-harness-text-helpers.js +459 -25
- package/src/inference/browser-harness.d.ts +7 -28
- package/src/inference/browser-harness.js +556 -79
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +31 -0
- package/src/inference/kv-cache/base.js +125 -2
- package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
- package/src/inference/kv-cache/mixed-geometry.js +121 -4
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
- package/src/inference/pipelines/text/attention/projections.js +398 -71
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +542 -56
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +343 -50
- package/src/inference/pipelines/text/attention/types.d.ts +7 -1
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +25 -26
- package/src/inference/pipelines/text/config.js +304 -25
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +25 -29
- package/src/inference/pipelines/text/embed.js +16 -45
- package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
- package/src/inference/pipelines/text/execution-plan.js +21 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
- package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
- package/src/inference/pipelines/text/execution-v1.js +461 -158
- package/src/inference/pipelines/text/ffn/dense.js +214 -6
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +84 -17
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +9 -7
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
- package/src/inference/pipelines/text/generator-steps.js +275 -79
- package/src/inference/pipelines/text/generator.d.ts +38 -5
- package/src/inference/pipelines/text/generator.js +1259 -496
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +13 -3
- package/src/inference/pipelines/text/init.js +213 -725
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +373 -558
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +419 -66
- package/src/inference/pipelines/text/logits/index.d.ts +3 -3
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +42 -4
- package/src/inference/pipelines/text/ops.js +20 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +119 -31
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +153 -9
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
- package/src/inference/pipelines/text/types.d.ts +92 -81
- package/src/inference/pipelines/text/weights.d.ts +4 -52
- package/src/inference/pipelines/text/weights.js +6 -56
- package/src/inference/pipelines/text.d.ts +33 -1
- package/src/inference/pipelines/text.js +375 -55
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +341 -174
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +14 -0
- package/src/loader/doppler-loader.js +378 -25
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +276 -11
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +61 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +52 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +36 -8
- package/src/loader/tensors/tensor-loader.js +521 -22
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +13 -42
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +57 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +162 -10
- package/src/rules/inference/capability-transforms.rules.json +111 -0
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +117 -16
- package/src/rules/inference/ffn.rules.json +24 -2
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +17 -0
- package/src/rules/rule-registry.js +59 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +241 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +31 -4
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +53 -9
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +8 -3
- package/src/tooling/command-api.d.ts +22 -1
- package/src/tooling/command-api.js +11 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +14 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +3 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +571 -37
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +107 -17
- package/src/tooling/node-command-runner.js +43 -2
- package/src/tooling/node-convert-worker-pool.js +15 -0
- package/src/tooling/node-convert-worker.js +11 -1
- package/src/tooling/node-converter.js +102 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +112 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +10 -5
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +110 -8
- package/src/tooling/source-runtime-bundle.d.ts +3 -0
- package/src/tooling/source-runtime-bundle.js +30 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +25 -12
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -9,14 +9,16 @@ import { f16ToF32, convertBF16ToF32GPU, shouldDequantizeToF16, applyBufferLayout
|
|
|
9
9
|
import { QK_K, Q4K_BLOCK_BYTES, Q6K_BLOCK_BYTES } from '../quantization-constants.js';
|
|
10
10
|
import { log, trace as debugTrace } from '../../debug/index.js';
|
|
11
11
|
import { selectRuleValue } from '../../rules/rule-registry.js';
|
|
12
|
-
import { dequantizeQ4KM, dequantizeQ4KMRowWise } from '../../converter/quantizer.js';
|
|
12
|
+
import { dequantizeQ4KM, dequantizeQ4KMRowWise, float32ToFloat16 } from '../../converter/quantizer.js';
|
|
13
13
|
import { hasSourceTransform } from './source-transform.js';
|
|
14
|
+
import { loadFunctionalDescriptor } from './functional-descriptor-loader.js';
|
|
14
15
|
|
|
15
16
|
// ============================================================================
|
|
16
17
|
// Q4K Detection
|
|
17
18
|
// ============================================================================
|
|
18
19
|
|
|
19
20
|
let loggedF32UpcastNonMatmul = false;
|
|
21
|
+
let loggedQ4KLimitFallback = false;
|
|
20
22
|
|
|
21
23
|
function isGpuBufferInstance(value) {
|
|
22
24
|
return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
|
|
@@ -76,6 +78,28 @@ function toUint8View(data) {
|
|
|
76
78
|
return new Uint8Array(data);
|
|
77
79
|
}
|
|
78
80
|
|
|
81
|
+
function toUint16View(data, label) {
|
|
82
|
+
const bytes = toUint8View(data);
|
|
83
|
+
if (bytes.byteLength % 2 !== 0) {
|
|
84
|
+
throw new Error(`${label}: byte length must be divisible by 2.`);
|
|
85
|
+
}
|
|
86
|
+
if (bytes.byteOffset % 2 === 0) {
|
|
87
|
+
return new Uint16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2);
|
|
88
|
+
}
|
|
89
|
+
return new Uint16Array(bytes.slice().buffer);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function toFloat32View(data, label) {
|
|
93
|
+
const bytes = toUint8View(data);
|
|
94
|
+
if (bytes.byteLength % 4 !== 0) {
|
|
95
|
+
throw new Error(`${label}: byte length must be divisible by 4.`);
|
|
96
|
+
}
|
|
97
|
+
if (bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength) {
|
|
98
|
+
return new Float32Array(bytes.buffer);
|
|
99
|
+
}
|
|
100
|
+
return new Float32Array(bytes.slice().buffer);
|
|
101
|
+
}
|
|
102
|
+
|
|
79
103
|
function resolveInputByteLength(data, fallbackSize) {
|
|
80
104
|
if (data instanceof Uint8Array) return data.byteLength;
|
|
81
105
|
if (ArrayBuffer.isView(data)) return data.byteLength;
|
|
@@ -107,12 +131,203 @@ function getShapeElementCount(shape) {
|
|
|
107
131
|
return shape.reduce((product, value) => product * value, 1);
|
|
108
132
|
}
|
|
109
133
|
|
|
134
|
+
function getStorageCompanion(shardData, location, name, role) {
|
|
135
|
+
const companion = shardData?.storageCompanions?.[role];
|
|
136
|
+
if (!companion || !(companion.bytes instanceof Uint8Array)) {
|
|
137
|
+
throw new Error(
|
|
138
|
+
`W4A16 tensor "${name}" is missing required storage companion "${role}".`
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
const declared = Array.isArray(location?.storage?.companions)
|
|
142
|
+
? location.storage.companions.find((entry) => entry.role === role)
|
|
143
|
+
: null;
|
|
144
|
+
if (declared && companion.tensorId !== declared.tensorId) {
|
|
145
|
+
throw new Error(
|
|
146
|
+
`W4A16 tensor "${name}" companion "${role}" resolved to "${companion.tensorId}", expected "${declared.tensorId}".`
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
return companion;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function readW4A16LogicalShape(companion, fallbackShape, name) {
|
|
153
|
+
const location = companion.location ?? null;
|
|
154
|
+
const bytes = companion.bytes;
|
|
155
|
+
const dtype = String(location?.dtype || '').toUpperCase();
|
|
156
|
+
if (!Array.isArray(location?.shape) || location.shape.length !== 1 || location.shape[0] !== 2) {
|
|
157
|
+
throw new Error(`W4A16 tensor "${name}" shape companion must have shape [2].`);
|
|
158
|
+
}
|
|
159
|
+
if (dtype === 'I64') {
|
|
160
|
+
if (bytes.byteLength !== 16) {
|
|
161
|
+
throw new Error(`W4A16 tensor "${name}" I64 shape companion must be 16 bytes.`);
|
|
162
|
+
}
|
|
163
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
164
|
+
return [Number(view.getBigInt64(0, true)), Number(view.getBigInt64(8, true))];
|
|
165
|
+
}
|
|
166
|
+
if (dtype === 'I32' || dtype === 'U32') {
|
|
167
|
+
if (bytes.byteLength !== 8) {
|
|
168
|
+
throw new Error(`W4A16 tensor "${name}" ${dtype} shape companion must be 8 bytes.`);
|
|
169
|
+
}
|
|
170
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
171
|
+
const read = dtype === 'I32'
|
|
172
|
+
? (offset) => view.getInt32(offset, true)
|
|
173
|
+
: (offset) => view.getUint32(offset, true);
|
|
174
|
+
return [read(0), read(4)];
|
|
175
|
+
}
|
|
176
|
+
if (Array.isArray(fallbackShape) && fallbackShape.length === 2) {
|
|
177
|
+
return fallbackShape;
|
|
178
|
+
}
|
|
179
|
+
throw new Error(`W4A16 tensor "${name}" has unsupported shape companion dtype "${location?.dtype}".`);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function assertW4A16Shape(shape, fallbackShape, name) {
|
|
183
|
+
if (!Array.isArray(shape) || shape.length !== 2) {
|
|
184
|
+
throw new Error(`W4A16 tensor "${name}" logical shape must be 2D.`);
|
|
185
|
+
}
|
|
186
|
+
const rows = Number(shape[0]);
|
|
187
|
+
const cols = Number(shape[1]);
|
|
188
|
+
if (!Number.isInteger(rows) || rows <= 0 || !Number.isInteger(cols) || cols <= 0) {
|
|
189
|
+
throw new Error(`W4A16 tensor "${name}" has invalid logical shape ${JSON.stringify(shape)}.`);
|
|
190
|
+
}
|
|
191
|
+
if (Array.isArray(fallbackShape) && fallbackShape.length === 2) {
|
|
192
|
+
if (rows !== fallbackShape[0] || cols !== fallbackShape[1]) {
|
|
193
|
+
throw new Error(
|
|
194
|
+
`W4A16 tensor "${name}" shape companion [${rows},${cols}] does not match manifest shape [${fallbackShape.join(',')}].`
|
|
195
|
+
);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
return [rows, cols];
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function bf16ToF32(bits) {
|
|
202
|
+
const floats = new Float32Array(1);
|
|
203
|
+
const uints = new Uint32Array(floats.buffer);
|
|
204
|
+
uints[0] = (bits & 0xffff) << 16;
|
|
205
|
+
return floats[0];
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function readOffsetBinaryInt4(byte, highNibble) {
|
|
209
|
+
const value = highNibble ? ((byte >> 4) & 0x0f) : (byte & 0x0f);
|
|
210
|
+
return value - 8;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function readW4A16Scales(companion, expectedScales, name) {
|
|
214
|
+
const dtype = String(companion.location?.dtype || '').toUpperCase();
|
|
215
|
+
if (dtype === 'F16') {
|
|
216
|
+
const packed = toUint16View(companion.bytes, `W4A16 scales for ${name}`);
|
|
217
|
+
if (packed.length !== expectedScales) {
|
|
218
|
+
throw new Error(
|
|
219
|
+
`W4A16 tensor "${name}" scale count ${packed.length} does not match expected ${expectedScales}.`
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
const scales = new Float32Array(packed.length);
|
|
223
|
+
for (let i = 0; i < packed.length; i += 1) {
|
|
224
|
+
scales[i] = f16ToF32(packed[i]);
|
|
225
|
+
}
|
|
226
|
+
return scales;
|
|
227
|
+
}
|
|
228
|
+
if (dtype === 'BF16') {
|
|
229
|
+
const packed = toUint16View(companion.bytes, `W4A16 scales for ${name}`);
|
|
230
|
+
if (packed.length !== expectedScales) {
|
|
231
|
+
throw new Error(
|
|
232
|
+
`W4A16 tensor "${name}" scale count ${packed.length} does not match expected ${expectedScales}.`
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
const scales = new Float32Array(packed.length);
|
|
236
|
+
for (let i = 0; i < packed.length; i += 1) {
|
|
237
|
+
scales[i] = bf16ToF32(packed[i]);
|
|
238
|
+
}
|
|
239
|
+
return scales;
|
|
240
|
+
}
|
|
241
|
+
if (dtype === 'F32') {
|
|
242
|
+
const scales = toFloat32View(companion.bytes, `W4A16 scales for ${name}`);
|
|
243
|
+
if (scales.length !== expectedScales) {
|
|
244
|
+
throw new Error(
|
|
245
|
+
`W4A16 tensor "${name}" scale count ${scales.length} does not match expected ${expectedScales}.`
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
return scales;
|
|
249
|
+
}
|
|
250
|
+
throw new Error(`W4A16 tensor "${name}" has unsupported scale companion dtype "${companion.location?.dtype}".`);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function validateW4A16PackedStorage(shardData, location, name) {
|
|
254
|
+
const scaleCompanion = getStorageCompanion(shardData, location, name, 'scales');
|
|
255
|
+
const shapeCompanion = getStorageCompanion(shardData, location, name, 'shape');
|
|
256
|
+
const [rows, cols] = assertW4A16Shape(
|
|
257
|
+
readW4A16LogicalShape(shapeCompanion, location.shape, name),
|
|
258
|
+
location.shape,
|
|
259
|
+
name
|
|
260
|
+
);
|
|
261
|
+
const groupsPerRow = Math.ceil(cols / 32);
|
|
262
|
+
const expectedPackedBytes = rows * groupsPerRow * 16;
|
|
263
|
+
if (shardData.byteLength !== expectedPackedBytes) {
|
|
264
|
+
throw new Error(
|
|
265
|
+
`W4A16 tensor "${name}" packed byte length ${shardData.byteLength} does not match expected ${expectedPackedBytes}.`
|
|
266
|
+
);
|
|
267
|
+
}
|
|
268
|
+
const scaleDtype = String(scaleCompanion.location?.dtype || '').toUpperCase();
|
|
269
|
+
const scaleBytesPerElement = scaleDtype === 'F32'
|
|
270
|
+
? 4
|
|
271
|
+
: (scaleDtype === 'F16' || scaleDtype === 'BF16' ? 2 : null);
|
|
272
|
+
if (scaleBytesPerElement == null) {
|
|
273
|
+
throw new Error(`W4A16 tensor "${name}" has unsupported scale companion dtype "${scaleCompanion.location?.dtype}".`);
|
|
274
|
+
}
|
|
275
|
+
const expectedScaleBytes = rows * groupsPerRow * scaleBytesPerElement;
|
|
276
|
+
if (scaleCompanion.bytes.byteLength !== expectedScaleBytes) {
|
|
277
|
+
throw new Error(
|
|
278
|
+
`W4A16 tensor "${name}" scale byte length ${scaleCompanion.bytes.byteLength} does not match expected ${expectedScaleBytes}.`
|
|
279
|
+
);
|
|
280
|
+
}
|
|
281
|
+
return {
|
|
282
|
+
rows,
|
|
283
|
+
cols,
|
|
284
|
+
groupsPerRow,
|
|
285
|
+
scaleDtype: scaleDtype.toLowerCase(),
|
|
286
|
+
scaleBytes: scaleCompanion.bytes,
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function dequantizeW4A16ToF16(shardData, location, name) {
|
|
291
|
+
const scaleCompanion = getStorageCompanion(shardData, location, name, 'scales');
|
|
292
|
+
const shapeCompanion = getStorageCompanion(shardData, location, name, 'shape');
|
|
293
|
+
const [rows, cols] = assertW4A16Shape(
|
|
294
|
+
readW4A16LogicalShape(shapeCompanion, location.shape, name),
|
|
295
|
+
location.shape,
|
|
296
|
+
name
|
|
297
|
+
);
|
|
298
|
+
const groupsPerRow = Math.ceil(cols / 32);
|
|
299
|
+
const expectedPackedBytes = rows * groupsPerRow * 16;
|
|
300
|
+
if (shardData.byteLength !== expectedPackedBytes) {
|
|
301
|
+
throw new Error(
|
|
302
|
+
`W4A16 tensor "${name}" packed byte length ${shardData.byteLength} does not match expected ${expectedPackedBytes}.`
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
const expectedScales = rows * groupsPerRow;
|
|
306
|
+
const scales = readW4A16Scales(scaleCompanion, expectedScales, name);
|
|
307
|
+
const out = new Uint16Array(rows * cols);
|
|
308
|
+
for (let row = 0; row < rows; row += 1) {
|
|
309
|
+
for (let group = 0; group < groupsPerRow; group += 1) {
|
|
310
|
+
const scale = scales[(row * groupsPerRow) + group];
|
|
311
|
+
const packedOffset = ((row * groupsPerRow) + group) * 16;
|
|
312
|
+
for (let lane = 0; lane < 32; lane += 1) {
|
|
313
|
+
const col = (group * 32) + lane;
|
|
314
|
+
if (col >= cols) break;
|
|
315
|
+
const byte = shardData[packedOffset + Math.floor(lane / 2)];
|
|
316
|
+
const quant = readOffsetBinaryInt4(byte, (lane % 2) === 1);
|
|
317
|
+
out[(row * cols) + col] = float32ToFloat16(quant * scale);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
|
|
322
|
+
}
|
|
323
|
+
|
|
110
324
|
|
|
111
325
|
export function isPackedQ4K(location) {
|
|
112
|
-
if (!Array.isArray(location.shape) || location.shape.length
|
|
326
|
+
if (!Array.isArray(location.shape) || location.shape.length < 2) {
|
|
113
327
|
return false;
|
|
114
328
|
}
|
|
115
|
-
const
|
|
329
|
+
const cols = location.shape[location.shape.length - 1];
|
|
330
|
+
const rows = location.shape.slice(0, -1).reduce((a, b) => a * b, 1);
|
|
116
331
|
const expectedRowwise = rows * Math.ceil(cols / QK_K) * Q4K_BLOCK_BYTES;
|
|
117
332
|
return location.size < expectedRowwise;
|
|
118
333
|
}
|
|
@@ -128,7 +343,10 @@ function isEmbeddingRole(location) {
|
|
|
128
343
|
|
|
129
344
|
export function shouldUseFusedQ4K(location, config) {
|
|
130
345
|
if (!config.useFusedQ4K) return false;
|
|
346
|
+
return canUseFusedQ4KStorage(location, config);
|
|
347
|
+
}
|
|
131
348
|
|
|
349
|
+
function canUseFusedQ4KStorage(location, config) {
|
|
132
350
|
const caps = config.gpuCapabilities || getKernelCapabilities();
|
|
133
351
|
if (!caps?.hasSubgroups) return false;
|
|
134
352
|
|
|
@@ -141,6 +359,125 @@ export function shouldUseFusedQ4K(location, config) {
|
|
|
141
359
|
return true;
|
|
142
360
|
}
|
|
143
361
|
|
|
362
|
+
export function isLiteRTAffineInt4FusedEligible(location, config) {
|
|
363
|
+
const caps = config?.gpuCapabilities || getKernelCapabilities();
|
|
364
|
+
if (caps?.hasF16 !== true) return false;
|
|
365
|
+
|
|
366
|
+
if (!Array.isArray(location?.shape) || location.shape.length !== 2) return false;
|
|
367
|
+
if (!location?.role) return false;
|
|
368
|
+
if (isEmbeddingRole(location)) return false;
|
|
369
|
+
if (location.role !== 'matmul' && location.role !== 'lm_head') return false;
|
|
370
|
+
if (!shouldDequantizeToF16(location)) return false;
|
|
371
|
+
|
|
372
|
+
const transform = location.sourceTransform;
|
|
373
|
+
if (!transform || typeof transform !== 'object') return false;
|
|
374
|
+
const sourceDtype = String(transform.sourceDtype || '').toUpperCase();
|
|
375
|
+
const targetDtype = String(transform.targetDtype || '').toUpperCase();
|
|
376
|
+
const locationDtype = String(location.dtype || '').toUpperCase();
|
|
377
|
+
const storageEncoding = String(transform.storageEncoding || '').toLowerCase();
|
|
378
|
+
const scale = Number(transform.scale);
|
|
379
|
+
const zeroPoint = Number(transform.zeroPoint);
|
|
380
|
+
const storageEncodingSupported = storageEncoding === 'signed' || storageEncoding === 'offset_binary';
|
|
381
|
+
|
|
382
|
+
return transform.kind === 'affine_dequant'
|
|
383
|
+
&& transform.scheme === 'per_tensor_affine'
|
|
384
|
+
&& sourceDtype === 'INT4'
|
|
385
|
+
&& targetDtype === 'F16'
|
|
386
|
+
&& locationDtype === 'F16'
|
|
387
|
+
&& storageEncodingSupported
|
|
388
|
+
&& Number.isFinite(scale)
|
|
389
|
+
&& Math.abs(scale - 0.0625) <= Number.EPSILON
|
|
390
|
+
&& Number.isSafeInteger(zeroPoint)
|
|
391
|
+
&& zeroPoint === 0;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
export function isW4A16FusedEligible(location, config) {
|
|
395
|
+
const caps = config?.gpuCapabilities || getKernelCapabilities();
|
|
396
|
+
if (caps?.hasF16 !== true) return false;
|
|
397
|
+
|
|
398
|
+
if (!Array.isArray(location?.shape) || location.shape.length !== 2) return false;
|
|
399
|
+
if (String(location?.dtype || '').toUpperCase() !== 'W4A16') return false;
|
|
400
|
+
if (!location?.role) return false;
|
|
401
|
+
if (isEmbeddingRole(location)) return false;
|
|
402
|
+
if (location.role !== 'matmul' && location.role !== 'lm_head') return false;
|
|
403
|
+
if (!shouldDequantizeToF16(location)) return false;
|
|
404
|
+
if (location?.storage?.packing !== 'w4a16') return false;
|
|
405
|
+
if (!Array.isArray(location?.storage?.companions)) return false;
|
|
406
|
+
|
|
407
|
+
const hasScales = location.storage.companions.some((entry) => entry.role === 'scales');
|
|
408
|
+
const hasShape = location.storage.companions.some((entry) => entry.role === 'shape');
|
|
409
|
+
return hasScales && hasShape;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
function getQ4KDenseMaterializedSizeBytes(location, config) {
|
|
413
|
+
if (!Array.isArray(location.shape) || location.shape.length === 0) {
|
|
414
|
+
return null;
|
|
415
|
+
}
|
|
416
|
+
const elementCount = getShapeElementCount(location.shape);
|
|
417
|
+
if (!Number.isFinite(elementCount) || elementCount <= 0) {
|
|
418
|
+
return null;
|
|
419
|
+
}
|
|
420
|
+
const outputDtype = getQ4KOutputDtype(location, config);
|
|
421
|
+
const bytesPerElement = outputDtype === 'f16' ? 2 : 4;
|
|
422
|
+
return elementCount * bytesPerElement;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
function getMaxStorageBufferBindingSize() {
|
|
426
|
+
const device = getDevice();
|
|
427
|
+
const maxStorage = device?.limits?.maxStorageBufferBindingSize;
|
|
428
|
+
return Number.isFinite(maxStorage) && maxStorage > 0 ? maxStorage : null;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
function resolveQ4KLimitFallback(location, config) {
|
|
432
|
+
if (location?.dtype !== 'Q4_K_M' && location?.dtype !== 'Q4_K') {
|
|
433
|
+
return {
|
|
434
|
+
denseExceedsBindingLimit: false,
|
|
435
|
+
limitFallbackEligible: false,
|
|
436
|
+
denseSizeBytes: null,
|
|
437
|
+
maxBindingSizeBytes: null,
|
|
438
|
+
};
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
const denseSizeBytes = getQ4KDenseMaterializedSizeBytes(location, config);
|
|
442
|
+
const maxBindingSizeBytes = getMaxStorageBufferBindingSize();
|
|
443
|
+
const denseExceedsBindingLimit = (
|
|
444
|
+
denseSizeBytes != null
|
|
445
|
+
&& maxBindingSizeBytes != null
|
|
446
|
+
&& denseSizeBytes > maxBindingSizeBytes
|
|
447
|
+
);
|
|
448
|
+
const packedSizeBytes = Number.isFinite(location.size) ? location.size : null;
|
|
449
|
+
const packedFitsBindingLimit = (
|
|
450
|
+
packedSizeBytes != null
|
|
451
|
+
&& maxBindingSizeBytes != null
|
|
452
|
+
&& packedSizeBytes <= maxBindingSizeBytes
|
|
453
|
+
);
|
|
454
|
+
const limitFallbackEligible = (
|
|
455
|
+
denseExceedsBindingLimit
|
|
456
|
+
&& config.keepF32Weights !== true
|
|
457
|
+
&& packedFitsBindingLimit
|
|
458
|
+
&& canUseFusedQ4KStorage(location, config)
|
|
459
|
+
);
|
|
460
|
+
|
|
461
|
+
return {
|
|
462
|
+
denseExceedsBindingLimit,
|
|
463
|
+
limitFallbackEligible,
|
|
464
|
+
denseSizeBytes,
|
|
465
|
+
maxBindingSizeBytes,
|
|
466
|
+
};
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
function logQ4KLimitFallbackOnce(name, fallback) {
|
|
470
|
+
if (loggedQ4KLimitFallback) {
|
|
471
|
+
return;
|
|
472
|
+
}
|
|
473
|
+
loggedQ4KLimitFallback = true;
|
|
474
|
+
log.warn(
|
|
475
|
+
'Loader',
|
|
476
|
+
`Q4K dense materialization for "${name}" would require ${fallback.denseSizeBytes} bytes, ` +
|
|
477
|
+
`exceeding maxStorageBufferBindingSize=${fallback.maxBindingSizeBytes}; retaining packed Q4K for fused matmul.`
|
|
478
|
+
);
|
|
479
|
+
}
|
|
480
|
+
|
|
144
481
|
// ============================================================================
|
|
145
482
|
// Dtype Output Selection
|
|
146
483
|
// ============================================================================
|
|
@@ -220,6 +557,80 @@ export async function loadQ4KFused(shardData, location, name) {
|
|
|
220
557
|
}
|
|
221
558
|
}
|
|
222
559
|
|
|
560
|
+
export async function loadLiteRTInt4Fused(shardData, location, name, config = null) {
|
|
561
|
+
if (isGpuBufferInstance(shardData)) {
|
|
562
|
+
throw new Error(
|
|
563
|
+
`LiteRT INT4 tensor "${name}" requires raw packed source bytes before GPU upload.`
|
|
564
|
+
);
|
|
565
|
+
}
|
|
566
|
+
if (!isLiteRTAffineInt4FusedEligible(location, config ?? { gpuCapabilities: getKernelCapabilities() })) {
|
|
567
|
+
throw new Error(
|
|
568
|
+
`LiteRT INT4 tensor "${name}" does not match the fused fixed-affine contract ` +
|
|
569
|
+
'(INT4 -> F16, storageEncoding=signed|offset_binary, per_tensor_affine, scale=0.0625, zeroPoint=0, 2D matmul/lm_head role).'
|
|
570
|
+
);
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
const [rows, cols] = location.shape;
|
|
574
|
+
const expectedBytes = rows * Math.ceil(cols / 2);
|
|
575
|
+
const actualBytes = resolveInputByteLength(shardData, location.size);
|
|
576
|
+
if (actualBytes !== expectedBytes) {
|
|
577
|
+
throw new Error(
|
|
578
|
+
`LiteRT INT4 tensor "${name}" packed byte size mismatch. ` +
|
|
579
|
+
`Expected ${expectedBytes} bytes for shape [${rows},${cols}], got ${actualBytes}.`
|
|
580
|
+
);
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
const device = getDevice();
|
|
584
|
+
const buffer = acquireAlignedBuffer(actualBytes, `litert_int4_${name}`);
|
|
585
|
+
try {
|
|
586
|
+
writeBufferAligned(device, buffer, shardData);
|
|
587
|
+
return {
|
|
588
|
+
data: createWeightBuffer(buffer, 'litert_int4', 'row', location.shape, name, null, {
|
|
589
|
+
storageEncoding: String(location.sourceTransform.storageEncoding).toLowerCase(),
|
|
590
|
+
}),
|
|
591
|
+
allocatedBuffers: [buffer],
|
|
592
|
+
};
|
|
593
|
+
} catch (error) {
|
|
594
|
+
releaseBuffer(buffer);
|
|
595
|
+
throw error;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
export async function loadW4A16Fused(shardData, location, name, config = null) {
|
|
600
|
+
if (isGpuBufferInstance(shardData)) {
|
|
601
|
+
throw new Error(
|
|
602
|
+
`W4A16 tensor "${name}" requires raw packed source bytes before GPU upload.`
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
if (!isW4A16FusedEligible(location, config ?? { gpuCapabilities: getKernelCapabilities() })) {
|
|
606
|
+
throw new Error(
|
|
607
|
+
`W4A16 tensor "${name}" does not match the fused packed contract ` +
|
|
608
|
+
'(dtype=W4A16, storage.packing=w4a16, scales+shape companions, 2D matmul/lm_head role).'
|
|
609
|
+
);
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
const storage = validateW4A16PackedStorage(shardData, location, name);
|
|
613
|
+
const device = getDevice();
|
|
614
|
+
const weightBuffer = acquireAlignedBuffer(shardData.byteLength, `w4a16_${name}`);
|
|
615
|
+
const scaleBuffer = acquireAlignedBuffer(storage.scaleBytes.byteLength, `w4a16_scales_${name}`);
|
|
616
|
+
try {
|
|
617
|
+
writeBufferAligned(device, weightBuffer, shardData);
|
|
618
|
+
writeBufferAligned(device, scaleBuffer, storage.scaleBytes);
|
|
619
|
+
return {
|
|
620
|
+
data: createWeightBuffer(weightBuffer, 'w4a16', 'row', location.shape, name, null, {
|
|
621
|
+
scaleBuffer,
|
|
622
|
+
scaleDtype: storage.scaleDtype,
|
|
623
|
+
groupsPerRow: storage.groupsPerRow,
|
|
624
|
+
}),
|
|
625
|
+
allocatedBuffers: [weightBuffer, scaleBuffer],
|
|
626
|
+
};
|
|
627
|
+
} catch (error) {
|
|
628
|
+
releaseBuffer(weightBuffer);
|
|
629
|
+
releaseBuffer(scaleBuffer);
|
|
630
|
+
throw error;
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
|
|
223
634
|
async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name, config) {
|
|
224
635
|
let dequantized = null;
|
|
225
636
|
const outputDtype = getQ4KOutputDtype(location, config);
|
|
@@ -268,16 +679,23 @@ async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name,
|
|
|
268
679
|
}
|
|
269
680
|
|
|
270
681
|
let numBlocks = null;
|
|
682
|
+
let rowCount = null;
|
|
683
|
+
let blocksPerRow = null;
|
|
684
|
+
let denseElementCount = null;
|
|
271
685
|
let dequantizedTensor;
|
|
272
686
|
if (needsRowwise) {
|
|
273
|
-
|
|
687
|
+
rowCount = location.shape.slice(0, -1).reduce((a, b) => a * b, 1);
|
|
688
|
+
blocksPerRow = Math.ceil(K / QK_K);
|
|
689
|
+
numBlocks = rowCount * blocksPerRow;
|
|
690
|
+
denseElementCount = rowCount * K;
|
|
274
691
|
debugTrace.loader(
|
|
275
|
-
`Dequantizing ${name} (row-wise): [${
|
|
692
|
+
`Dequantizing ${name} (row-wise): [${rowCount},${K}], K not 256-aligned, ` +
|
|
276
693
|
`outputDtype=${outputDtype}`
|
|
277
694
|
);
|
|
278
|
-
dequantizedTensor = await dequantizeRowwise(quantBuffer,
|
|
695
|
+
dequantizedTensor = await dequantizeRowwise(quantBuffer, rowCount, K, { outputDtype });
|
|
279
696
|
} else {
|
|
280
697
|
numBlocks = Math.ceil(location.size / Q4K_BLOCK_BYTES);
|
|
698
|
+
denseElementCount = numBlocks * QK_K;
|
|
281
699
|
debugTrace.loader(
|
|
282
700
|
`Dequantizing ${name}: size=${location.size}, numBlocks=${numBlocks}, ` +
|
|
283
701
|
`outputDtype=${outputDtype}, expectedOutput=${numBlocks * QK_K * (outputDtype === 'f16' ? 2 : 4)}`
|
|
@@ -296,9 +714,9 @@ async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name,
|
|
|
296
714
|
if (isProbeTarget) {
|
|
297
715
|
try {
|
|
298
716
|
const bytesPerElem = outputDtype === 'f16' ? 2 : 4;
|
|
299
|
-
const requestedOutputBytes =
|
|
717
|
+
const requestedOutputBytes = denseElementCount * bytesPerElem;
|
|
300
718
|
const sampleCount = paritySamples;
|
|
301
|
-
const readSize = Math.min(sampleCount * bytesPerElem, dequantized.size);
|
|
719
|
+
const readSize = Math.min(sampleCount * bytesPerElem, requestedOutputBytes, dequantized.size);
|
|
302
720
|
const gpuRaw = await readBuffer(dequantized, readSize);
|
|
303
721
|
const gpuBytes = gpuRaw instanceof ArrayBuffer
|
|
304
722
|
? new Uint8Array(gpuRaw)
|
|
@@ -334,7 +752,8 @@ async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name,
|
|
|
334
752
|
|
|
335
753
|
log.warn('DequantProbe',
|
|
336
754
|
`tensor="${name}" shape=[${location.shape}] ` +
|
|
337
|
-
`location.size=${location.size} numBlocks=${numBlocks}
|
|
755
|
+
`location.size=${location.size} numBlocks=${numBlocks} blocksPerRow=${blocksPerRow} ` +
|
|
756
|
+
`rows=${rowCount} K=${K} denseElements=${denseElementCount} outputDtype=${outputDtype} ` +
|
|
338
757
|
`bytesPerElem=${bytesPerElem} requestedOutputBytes=${requestedOutputBytes} bufSize=${dequantized.size} ` +
|
|
339
758
|
`runParity=true sampleCount=${sampleCount}`
|
|
340
759
|
);
|
|
@@ -389,7 +808,7 @@ export async function loadQ4KDequant(shardData, location, name, config) {
|
|
|
389
808
|
}
|
|
390
809
|
}
|
|
391
810
|
|
|
392
|
-
|
|
811
|
+
async function loadQ4KMixed(shardData, location, name, config) {
|
|
393
812
|
const canMaterializeMixed = shouldUseFusedQ4K(location, config)
|
|
394
813
|
&& config.q4kLayout === 'row';
|
|
395
814
|
if (!canMaterializeMixed) {
|
|
@@ -436,10 +855,10 @@ export async function loadQ4KMixed(shardData, location, name, config) {
|
|
|
436
855
|
|
|
437
856
|
function getQ4KCpuReferenceContext(shardData, location, config) {
|
|
438
857
|
const outputDtype = getQ4KOutputDtype(location, config);
|
|
439
|
-
const
|
|
440
|
-
const K =
|
|
441
|
-
const needsRowwise = is2DMatrix && K > 0 && K % QK_K !== 0;
|
|
858
|
+
const isMatrixLike = Array.isArray(location.shape) && location.shape.length >= 2;
|
|
859
|
+
const K = isMatrixLike ? location.shape[location.shape.length - 1] : 0;
|
|
442
860
|
const layout = getWeightLayout(location, config);
|
|
861
|
+
const needsRowwise = isMatrixLike && layout === 'row' && K > 0 && K % QK_K !== 0;
|
|
443
862
|
const eligible = outputDtype === 'f32'
|
|
444
863
|
&& !isGpuBufferInstance(shardData)
|
|
445
864
|
&& (!needsRowwise || layout === 'row');
|
|
@@ -655,12 +1074,58 @@ export async function loadFloat(shardData, location, name, config) {
|
|
|
655
1074
|
}
|
|
656
1075
|
}
|
|
657
1076
|
|
|
1077
|
+
export async function loadW4A16Dequant(shardData, location, name, config) {
|
|
1078
|
+
if (!config) {
|
|
1079
|
+
throw new Error('Tensor load config is required.');
|
|
1080
|
+
}
|
|
1081
|
+
if (isGpuBufferInstance(shardData)) {
|
|
1082
|
+
throw new Error(
|
|
1083
|
+
`W4A16 tensor "${name}" requires CPU-side storage companion materialization before GPU upload.`
|
|
1084
|
+
);
|
|
1085
|
+
}
|
|
1086
|
+
const f16Bytes = dequantizeW4A16ToF16(shardData, location, name);
|
|
1087
|
+
const device = getDevice();
|
|
1088
|
+
const buffer = acquireAlignedBuffer(f16Bytes.byteLength, `w4a16_dequant_${name}`);
|
|
1089
|
+
try {
|
|
1090
|
+
writeBufferAligned(device, buffer, f16Bytes);
|
|
1091
|
+
const layout = selectRuleValue('loader', 'weights', 'weightLayout', {
|
|
1092
|
+
layout: location.layout ?? null,
|
|
1093
|
+
useColumnWise: false,
|
|
1094
|
+
});
|
|
1095
|
+
if (shouldDequantizeToF16(location)) {
|
|
1096
|
+
return {
|
|
1097
|
+
data: createWeightBuffer(buffer, 'f16', layout, location.shape, name),
|
|
1098
|
+
allocatedBuffers: [buffer],
|
|
1099
|
+
};
|
|
1100
|
+
}
|
|
1101
|
+
return {
|
|
1102
|
+
data: applyBufferLayout(buffer, location, 'f16'),
|
|
1103
|
+
allocatedBuffers: [buffer],
|
|
1104
|
+
};
|
|
1105
|
+
} catch (error) {
|
|
1106
|
+
releaseBuffer(buffer);
|
|
1107
|
+
throw error;
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
|
|
658
1111
|
// ============================================================================
|
|
659
1112
|
// Main GPU Loading Entry Point
|
|
660
1113
|
// ============================================================================
|
|
661
1114
|
|
|
662
1115
|
|
|
663
1116
|
const GPU_LOADER_DISPATCH = {
|
|
1117
|
+
functional_descriptor: (shardData, location, name, config) => {
|
|
1118
|
+
debugTrace.loader(`Loading functional descriptor: ${name}`);
|
|
1119
|
+
return loadFunctionalDescriptor(shardData, location, name, config);
|
|
1120
|
+
},
|
|
1121
|
+
litert_int4_fused: (shardData, location, name, config) => {
|
|
1122
|
+
debugTrace.loader(`Loading LiteRT INT4 weight (fused): ${name} (size=${location.size})`);
|
|
1123
|
+
return loadLiteRTInt4Fused(shardData, location, name, config);
|
|
1124
|
+
},
|
|
1125
|
+
w4a16_fused: (shardData, location, name, config) => {
|
|
1126
|
+
debugTrace.loader(`Loading W4A16 weight (fused): ${name} (size=${location.size})`);
|
|
1127
|
+
return loadW4A16Fused(shardData, location, name, config);
|
|
1128
|
+
},
|
|
664
1129
|
q4k_mixed: (shardData, location, name, config) => loadQ4KMixed(shardData, location, name, config),
|
|
665
1130
|
q4k_fused: (shardData, location, name, _config) => {
|
|
666
1131
|
debugTrace.loader(`Loading Q4K weight (fused): ${name} (size=${location.size})`);
|
|
@@ -686,52 +1151,86 @@ const GPU_LOADER_DISPATCH = {
|
|
|
686
1151
|
}
|
|
687
1152
|
),
|
|
688
1153
|
q6k: (shardData, location, name, _config) => loadQ6K(shardData, location, name),
|
|
1154
|
+
w4a16_dequant_reference: (shardData, location, name, config) => loadW4A16Dequant(shardData, location, name, config),
|
|
689
1155
|
bf16: (shardData, location, name, config) => loadBF16(shardData, location, name, config),
|
|
690
1156
|
float: (shardData, location, name, config) => loadFloat(shardData, location, name, config),
|
|
1157
|
+
unsupported_packed_quantization: (_shardData, location, name, _config) => {
|
|
1158
|
+
throw new Error(
|
|
1159
|
+
`Unsupported packed quantization dtype "${location.dtype}" for tensor "${name}". ` +
|
|
1160
|
+
'Add a native loader and kernel path before enabling runtime execution.'
|
|
1161
|
+
);
|
|
1162
|
+
},
|
|
691
1163
|
};
|
|
692
1164
|
|
|
693
1165
|
export async function loadTensorToGPU(shardData, location, name, config) {
|
|
694
1166
|
const dtype = location.dtype;
|
|
695
1167
|
const useFusedQ4K = shouldUseFusedQ4K(location, config);
|
|
1168
|
+
const requiresFusedQ4KRole = Array.isArray(config?.q4kFusedRoles)
|
|
1169
|
+
&& config.q4kFusedRoles.includes(location.role);
|
|
696
1170
|
const caps = config?.gpuCapabilities || getKernelCapabilities();
|
|
697
1171
|
const platformId = getPlatformConfig()?.platform?.id ?? null;
|
|
698
1172
|
const q4kReferenceContext = getQ4KCpuReferenceContext(shardData, location, config);
|
|
699
1173
|
const q4kBasicBackendClass = platformId === 'basic'
|
|
700
1174
|
|| (caps?.hasSubgroups !== true && caps?.hasF16 !== true);
|
|
1175
|
+
const q4kLimitFallback = resolveQ4KLimitFallback(location, config);
|
|
1176
|
+
const litertAffineInt4FusedEligible = isLiteRTAffineInt4FusedEligible(location, { ...config, gpuCapabilities: caps });
|
|
1177
|
+
const w4a16FusedEligible = isW4A16FusedEligible(location, { ...config, gpuCapabilities: caps });
|
|
701
1178
|
const loaderPath = selectRuleValue('loader', 'tensorLoader', 'gpuLoaderPath', {
|
|
702
1179
|
dtype,
|
|
1180
|
+
role: location.role ?? null,
|
|
1181
|
+
litertAffineInt4FusedEligible,
|
|
1182
|
+
w4a16FusedEligible,
|
|
703
1183
|
useFusedQ4K,
|
|
1184
|
+
requiresFusedQ4KRole,
|
|
704
1185
|
q4kMaterializationMode: config.q4kMaterializationMode ?? 'dense',
|
|
705
1186
|
q4kCpuReferenceEligible: q4kReferenceContext.eligible,
|
|
706
1187
|
q4kBasicBackendClass,
|
|
1188
|
+
q4kDenseExceedsBindingLimit: q4kLimitFallback.denseExceedsBindingLimit,
|
|
1189
|
+
q4kLimitFallbackEligible: q4kLimitFallback.limitFallbackEligible,
|
|
707
1190
|
});
|
|
708
1191
|
const loader = GPU_LOADER_DISPATCH[loaderPath];
|
|
709
1192
|
if (!loader) {
|
|
710
1193
|
throw new Error(`Unknown GPU loader path: "${loaderPath}" for dtype "${dtype}"`);
|
|
711
1194
|
}
|
|
1195
|
+
if (loaderPath === 'q4k_fused' && q4kLimitFallback.limitFallbackEligible) {
|
|
1196
|
+
logQ4KLimitFallbackOnce(name, q4kLimitFallback);
|
|
1197
|
+
}
|
|
712
1198
|
return loader(shardData, location, name, config);
|
|
713
1199
|
}
|
|
714
1200
|
|
|
715
1201
|
|
|
716
1202
|
const CPU_LOADER_DISPATCH = {
|
|
1203
|
+
unsupported_functional_descriptor: (_shardData, _location, name) => {
|
|
1204
|
+
throw new Error(
|
|
1205
|
+
`FUNCTIONAL_DESCRIPTOR tensor "${name ?? 'unknown'}" requires GPU materialization through loadTensorToGPU.`
|
|
1206
|
+
);
|
|
1207
|
+
},
|
|
717
1208
|
raw: (shardData, _location) => shardData,
|
|
718
|
-
|
|
719
|
-
const
|
|
720
|
-
return
|
|
1209
|
+
w4a16_dequant_reference: (shardData, location) => {
|
|
1210
|
+
const f16Bytes = dequantizeW4A16ToF16(shardData, location, 'cpu');
|
|
1211
|
+
return convertF16ToF32CPU(toUint16View(f16Bytes, 'W4A16 CPU dequantized tensor load'));
|
|
721
1212
|
},
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
1213
|
+
unsupported_packed_quantization: (_shardData, location) => {
|
|
1214
|
+
throw new Error(
|
|
1215
|
+
`Unsupported packed quantization dtype "${location.dtype}" for CPU tensor load. ` +
|
|
1216
|
+
'Add a native loader before enabling runtime execution.'
|
|
1217
|
+
);
|
|
725
1218
|
},
|
|
726
|
-
|
|
1219
|
+
bf16_to_f32: (shardData, _location) => convertBF16ToF32CPU(
|
|
1220
|
+
toUint16View(shardData, 'BF16 CPU tensor load')
|
|
1221
|
+
),
|
|
1222
|
+
f16_to_f32: (shardData, _location) => convertF16ToF32CPU(
|
|
1223
|
+
toUint16View(shardData, 'F16 CPU tensor load')
|
|
1224
|
+
),
|
|
1225
|
+
f32: (shardData, _location) => toFloat32View(shardData, 'F32 CPU tensor load'),
|
|
727
1226
|
};
|
|
728
1227
|
|
|
729
|
-
export function loadTensorToCPU(shardData, location) {
|
|
1228
|
+
export function loadTensorToCPU(shardData, location, name = null) {
|
|
730
1229
|
const dtype = location.dtype;
|
|
731
1230
|
const loaderPath = selectRuleValue('loader', 'tensorLoader', 'cpuLoaderPath', { dtype });
|
|
732
1231
|
const loader = CPU_LOADER_DISPATCH[loaderPath];
|
|
733
1232
|
if (!loader) {
|
|
734
1233
|
throw new Error(`Unknown CPU loader path: "${loaderPath}" for dtype "${dtype}"`);
|
|
735
1234
|
}
|
|
736
|
-
return loader(shardData, location);
|
|
1235
|
+
return loader(shardData, location, name);
|
|
737
1236
|
}
|