doppler-gpu 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +210 -0
- package/README.md +63 -38
- package/assets/doppler.svg +57 -1601
- package/package.json +111 -32
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +486 -0
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +885 -744
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +469 -43
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +66 -10
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +69 -4
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +34 -50
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +35 -2
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +148 -3
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +8 -60
- package/src/client/wrap-pipeline-handle.d.ts +82 -0
- package/src/client/wrap-pipeline-handle.js +259 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +28 -157
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +163 -126
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +3593 -1146
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +3 -1
- package/src/config/merge-helpers.js +2 -1
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +65 -7
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +20 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +16 -0
- package/src/config/schema/converter.schema.js +11 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +13 -2
- package/src/config/schema/doppler.schema.js +140 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +39 -1
- package/src/config/schema/execution-v1.schema.js +35 -20
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +14 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +175 -2
- package/src/config/schema/manifest.schema.js +58 -3
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -9
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +183 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
- package/src/config/transforms/execution-graph-transforms.js +829 -168
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +31 -14
- package/src/converter/core.d.ts +65 -0
- package/src/converter/core.js +1153 -22
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +218 -5
- package/src/converter/quantizer.d.ts +24 -4
- package/src/converter/quantizer.js +92 -2
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +296 -38
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1733 -58
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +5 -14
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +192 -12
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +111 -2
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +10 -0
- package/src/formats/rdrr/index.js +10 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +26 -6
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +193 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +111 -2
- package/src/formats/rdrr/validation.js +260 -1
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +45 -23
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +130 -14
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +1 -8
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +458 -16
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +13 -120
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +84 -8
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +25 -22
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +77 -5
- package/src/gpu/kernels/index.js +55 -4
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +44 -2
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +172 -10
- package/src/gpu/kernels/matmul.d.ts +1 -29
- package/src/gpu/kernels/matmul.js +240 -38
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +1 -56
- package/src/gpu/kernels/moe.js +46 -33
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -12
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +6 -4
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +30 -7
- package/src/gpu/kernels/rmsnorm.js +49 -8
- package/src/gpu/kernels/rmsnorm.wgsl +15 -12
- package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +28 -21
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +8 -223
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +43 -0
- package/src/gpu/kernels/utils.js +81 -11
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +10 -12
- package/src/gpu/uniform-cache.js +25 -30
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
- package/src/inference/browser-harness-suite-helpers.js +525 -3
- package/src/inference/browser-harness-text-helpers.d.ts +89 -0
- package/src/inference/browser-harness-text-helpers.js +459 -25
- package/src/inference/browser-harness.d.ts +7 -28
- package/src/inference/browser-harness.js +556 -79
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +31 -0
- package/src/inference/kv-cache/base.js +125 -2
- package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
- package/src/inference/kv-cache/mixed-geometry.js +121 -4
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
- package/src/inference/pipelines/text/attention/projections.js +398 -71
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +542 -56
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +343 -50
- package/src/inference/pipelines/text/attention/types.d.ts +7 -1
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +25 -26
- package/src/inference/pipelines/text/config.js +304 -25
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +25 -29
- package/src/inference/pipelines/text/embed.js +16 -45
- package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
- package/src/inference/pipelines/text/execution-plan.js +21 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
- package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
- package/src/inference/pipelines/text/execution-v1.js +461 -158
- package/src/inference/pipelines/text/ffn/dense.js +214 -6
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +84 -17
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +9 -7
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
- package/src/inference/pipelines/text/generator-steps.js +275 -79
- package/src/inference/pipelines/text/generator.d.ts +38 -5
- package/src/inference/pipelines/text/generator.js +1259 -496
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +13 -3
- package/src/inference/pipelines/text/init.js +213 -725
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +373 -558
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +419 -66
- package/src/inference/pipelines/text/logits/index.d.ts +3 -3
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +42 -4
- package/src/inference/pipelines/text/ops.js +20 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +119 -31
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +153 -9
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
- package/src/inference/pipelines/text/types.d.ts +92 -81
- package/src/inference/pipelines/text/weights.d.ts +4 -52
- package/src/inference/pipelines/text/weights.js +6 -56
- package/src/inference/pipelines/text.d.ts +33 -1
- package/src/inference/pipelines/text.js +375 -55
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +341 -174
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +14 -0
- package/src/loader/doppler-loader.js +378 -25
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +276 -11
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +61 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +52 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +36 -8
- package/src/loader/tensors/tensor-loader.js +521 -22
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +13 -42
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +57 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +162 -10
- package/src/rules/inference/capability-transforms.rules.json +111 -0
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +117 -16
- package/src/rules/inference/ffn.rules.json +24 -2
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +17 -0
- package/src/rules/rule-registry.js +59 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +241 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +31 -4
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +53 -9
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +8 -3
- package/src/tooling/command-api.d.ts +22 -1
- package/src/tooling/command-api.js +11 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +14 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +3 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +571 -37
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +107 -17
- package/src/tooling/node-command-runner.js +43 -2
- package/src/tooling/node-convert-worker-pool.js +15 -0
- package/src/tooling/node-convert-worker.js +11 -1
- package/src/tooling/node-converter.js +102 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +112 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +10 -5
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +110 -8
- package/src/tooling/source-runtime-bundle.d.ts +3 -0
- package/src/tooling/source-runtime-bundle.js +30 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +25 -12
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
package/src/converter/core.js
CHANGED
|
@@ -38,7 +38,9 @@ import {
|
|
|
38
38
|
quantizeToQ4KM,
|
|
39
39
|
quantizeToQ4KMRowWise,
|
|
40
40
|
quantizeToQ4KMColumnWise,
|
|
41
|
+
quantizeToInt4PerRowSymmetric,
|
|
41
42
|
} from './quantizer.js';
|
|
43
|
+
import { cloneJsonValue } from '../utils/clone-json.js';
|
|
42
44
|
|
|
43
45
|
// ============================================================================
|
|
44
46
|
// Re-exports for Backward Compatibility
|
|
@@ -168,9 +170,77 @@ export function normalizeStorageQuant(value) {
|
|
|
168
170
|
if (lower === 'fp32' || lower === 'float32') return 'f32';
|
|
169
171
|
if (lower === 'bfloat16') return 'bf16';
|
|
170
172
|
if (lower === 'q4_k_m' || lower === 'q4km') return 'q4k';
|
|
173
|
+
if (lower === 'q4_0' || lower === 'q4-0') return 'q4_0';
|
|
174
|
+
if (
|
|
175
|
+
lower === 'w4a16-ct'
|
|
176
|
+
|| lower === 'w4a16_ct'
|
|
177
|
+
|| lower === 'compressed-tensors-w4a16'
|
|
178
|
+
|| lower === 'compressed_tensors_w4a16'
|
|
179
|
+
) return 'w4a16';
|
|
180
|
+
if (lower === 'wna8-o8' || lower === 'wna8_o8') return 'wna8o8';
|
|
171
181
|
return lower;
|
|
172
182
|
}
|
|
173
183
|
|
|
184
|
+
const SOURCE_PACKED_QUANT_DTYPES = new Set(['q4_0', 'w4a16', 'wna8o8']);
|
|
185
|
+
|
|
186
|
+
const SOURCE_PACKED_MANIFEST_DTYPES = {
|
|
187
|
+
q4_0: 'Q4_0',
|
|
188
|
+
w4a16: 'W4A16',
|
|
189
|
+
wna8o8: 'WNA8O8',
|
|
190
|
+
};
|
|
191
|
+
|
|
192
|
+
const SOURCE_PACKED_STORAGE_DESCRIPTORS = {
|
|
193
|
+
q4_0: {
|
|
194
|
+
packing: 'q4_0',
|
|
195
|
+
blockShape: [32],
|
|
196
|
+
blockBytes: 18,
|
|
197
|
+
},
|
|
198
|
+
w4a16: {
|
|
199
|
+
packing: 'w4a16',
|
|
200
|
+
blockShape: [32],
|
|
201
|
+
blockBytes: 16,
|
|
202
|
+
},
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
const COMPRESSED_TENSORS_W4A16_SUFFIXES = {
|
|
206
|
+
packed: '.weight_packed',
|
|
207
|
+
scale: '.weight_scale',
|
|
208
|
+
shape: '.weight_shape',
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
function cloneSourcePackedStorageDescriptor(targetQuant) {
|
|
212
|
+
const descriptor = SOURCE_PACKED_STORAGE_DESCRIPTORS[targetQuant];
|
|
213
|
+
if (!descriptor) return null;
|
|
214
|
+
return {
|
|
215
|
+
...descriptor,
|
|
216
|
+
blockShape: [...descriptor.blockShape],
|
|
217
|
+
...(Array.isArray(descriptor.companions)
|
|
218
|
+
? { companions: descriptor.companions.map((companion) => ({ ...companion })) }
|
|
219
|
+
: {}),
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function resolveExplicitRoleQuant(tensor, quantizationInfo) {
|
|
224
|
+
if (!quantizationInfo || typeof quantizationInfo !== 'object') {
|
|
225
|
+
return null;
|
|
226
|
+
}
|
|
227
|
+
const role = resolveTensorRole(tensor);
|
|
228
|
+
if (role === 'embedding') {
|
|
229
|
+
return normalizeStorageQuant(quantizationInfo.embeddings ?? null);
|
|
230
|
+
}
|
|
231
|
+
if (role === 'lm_head') {
|
|
232
|
+
return normalizeStorageQuant(
|
|
233
|
+
quantizationInfo.lmHead
|
|
234
|
+
?? quantizationInfo.embeddings
|
|
235
|
+
?? null
|
|
236
|
+
);
|
|
237
|
+
}
|
|
238
|
+
if (role === 'matmul' || role === 'expert' || role === 'router') {
|
|
239
|
+
return normalizeStorageQuant(quantizationInfo.weights ?? null);
|
|
240
|
+
}
|
|
241
|
+
return null;
|
|
242
|
+
}
|
|
243
|
+
|
|
174
244
|
export function resolveTensorTargetQuant(tensorOrName, fallbackQuant, quantizationInfo) {
|
|
175
245
|
const fallback = normalizeStorageQuant(fallbackQuant);
|
|
176
246
|
if (!quantizationInfo || typeof quantizationInfo !== 'object') {
|
|
@@ -212,30 +282,45 @@ function shouldExcludeTextOnlyTensor(name) {
|
|
|
212
282
|
const lower = name.toLowerCase();
|
|
213
283
|
return lower.startsWith('vision_tower.')
|
|
214
284
|
|| lower.startsWith('model.vision_tower.')
|
|
285
|
+
|| lower.startsWith('model.encoder.vision_tower.')
|
|
215
286
|
|| lower.startsWith('vision_model.')
|
|
216
287
|
|| lower.startsWith('model.vision_model.')
|
|
288
|
+
|| lower.startsWith('model.encoder.vision_model.')
|
|
217
289
|
|| lower.startsWith('visual.')
|
|
218
290
|
|| lower.startsWith('model.visual.')
|
|
291
|
+
|| lower.startsWith('model.encoder.visual.')
|
|
219
292
|
|| lower.startsWith('embed_vision.')
|
|
220
293
|
|| lower.startsWith('model.embed_vision.')
|
|
294
|
+
|| lower.startsWith('model.encoder.embed_vision.')
|
|
221
295
|
|| lower.startsWith('vision.')
|
|
222
296
|
|| lower.startsWith('model.vision.')
|
|
297
|
+
|| lower.startsWith('model.encoder.vision.')
|
|
223
298
|
|| lower.startsWith('vision_encoder.')
|
|
299
|
+
|| lower.startsWith('model.encoder.vision_encoder.')
|
|
224
300
|
|| lower.startsWith('image_encoder.')
|
|
301
|
+
|| lower.startsWith('model.encoder.image_encoder.')
|
|
225
302
|
|| lower.startsWith('image_tower.')
|
|
303
|
+
|| lower.startsWith('model.encoder.image_tower.')
|
|
226
304
|
|| lower.startsWith('image.')
|
|
227
305
|
|| lower.startsWith('model.image.')
|
|
306
|
+
|| lower.startsWith('model.encoder.image.')
|
|
228
307
|
|| lower.startsWith('audio_tower.')
|
|
229
308
|
|| lower.startsWith('model.audio_tower.')
|
|
309
|
+
|| lower.startsWith('model.encoder.audio_tower.')
|
|
230
310
|
|| lower.startsWith('audio_model.')
|
|
231
311
|
|| lower.startsWith('model.audio_model.')
|
|
312
|
+
|| lower.startsWith('model.encoder.audio_model.')
|
|
232
313
|
|| lower.startsWith('audio.')
|
|
233
314
|
|| lower.startsWith('model.audio.')
|
|
315
|
+
|| lower.startsWith('model.encoder.audio.')
|
|
234
316
|
|| lower.startsWith('audio_encoder.')
|
|
317
|
+
|| lower.startsWith('model.encoder.audio_encoder.')
|
|
235
318
|
|| lower.startsWith('multi_modal_projector.')
|
|
236
319
|
|| lower.startsWith('model.multi_modal_projector.')
|
|
320
|
+
|| lower.startsWith('model.encoder.multi_modal_projector.')
|
|
237
321
|
|| lower.startsWith('mm_projector.')
|
|
238
|
-
|| lower.startsWith('model.mm_projector.')
|
|
322
|
+
|| lower.startsWith('model.mm_projector.')
|
|
323
|
+
|| lower.startsWith('model.encoder.mm_projector.');
|
|
239
324
|
}
|
|
240
325
|
|
|
241
326
|
function resolveConversionTensors(model, converterConfig) {
|
|
@@ -255,7 +340,16 @@ function resolveConversionTensors(model, converterConfig) {
|
|
|
255
340
|
if (hasLanguageModelNamespace) {
|
|
256
341
|
return source.filter((tensor) => {
|
|
257
342
|
const lower = normalizeTensorName(tensor).toLowerCase();
|
|
258
|
-
|
|
343
|
+
// Keep top-level lm_head/output tensors alongside the language_model.*
|
|
344
|
+
// namespace. Multimodal HF models (e.g. Qwen 3.6-27B) place the
|
|
345
|
+
// language model body under model.language_model.* but expose the
|
|
346
|
+
// language modeling head at the bare top level (`lm_head.weight`).
|
|
347
|
+
// Dropping it leaves text-only conversion without an LM head and
|
|
348
|
+
// pipeline init fails at loadWeights.
|
|
349
|
+
return lower.startsWith('language_model.')
|
|
350
|
+
|| lower.startsWith('model.language_model.')
|
|
351
|
+
|| lower === 'lm_head.weight'
|
|
352
|
+
|| lower === 'model.lm_head.weight';
|
|
259
353
|
});
|
|
260
354
|
}
|
|
261
355
|
|
|
@@ -264,6 +358,283 @@ function resolveConversionTensors(model, converterConfig) {
|
|
|
264
358
|
));
|
|
265
359
|
}
|
|
266
360
|
|
|
361
|
+
function normalizePositiveIntegerShape(value, tensorName) {
|
|
362
|
+
if (!Array.isArray(value) || value.length === 0) {
|
|
363
|
+
throw new Error(`Compressed-tensors W4A16 tensor "${tensorName}" is missing a logical shape.`);
|
|
364
|
+
}
|
|
365
|
+
return value.map((entry, index) => {
|
|
366
|
+
const number = Number(entry);
|
|
367
|
+
if (!Number.isInteger(number) || number <= 0) {
|
|
368
|
+
throw new Error(
|
|
369
|
+
`Compressed-tensors W4A16 tensor "${tensorName}" has invalid shape[${index}]=${JSON.stringify(entry)}.`
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
return number;
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
function resolveCompressedTensorBaseName(name, suffix) {
|
|
377
|
+
return name.endsWith(suffix)
|
|
378
|
+
? name.slice(0, -suffix.length)
|
|
379
|
+
: null;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
function collectCompressedTensorsW4A16(tensors) {
|
|
383
|
+
const groups = new Map();
|
|
384
|
+
const groupFor = (baseName) => {
|
|
385
|
+
const existing = groups.get(baseName);
|
|
386
|
+
if (existing) return existing;
|
|
387
|
+
const created = {
|
|
388
|
+
baseName,
|
|
389
|
+
packed: null,
|
|
390
|
+
scale: null,
|
|
391
|
+
shape: null,
|
|
392
|
+
};
|
|
393
|
+
groups.set(baseName, created);
|
|
394
|
+
return created;
|
|
395
|
+
};
|
|
396
|
+
|
|
397
|
+
for (const tensor of tensors) {
|
|
398
|
+
const name = normalizeTensorName(tensor);
|
|
399
|
+
const packedBase = resolveCompressedTensorBaseName(name, COMPRESSED_TENSORS_W4A16_SUFFIXES.packed);
|
|
400
|
+
if (packedBase) {
|
|
401
|
+
groupFor(packedBase).packed = tensor;
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
404
|
+
const scaleBase = resolveCompressedTensorBaseName(name, COMPRESSED_TENSORS_W4A16_SUFFIXES.scale);
|
|
405
|
+
if (scaleBase) {
|
|
406
|
+
groupFor(scaleBase).scale = tensor;
|
|
407
|
+
continue;
|
|
408
|
+
}
|
|
409
|
+
const shapeBase = resolveCompressedTensorBaseName(name, COMPRESSED_TENSORS_W4A16_SUFFIXES.shape);
|
|
410
|
+
if (shapeBase) {
|
|
411
|
+
groupFor(shapeBase).shape = tensor;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
return groups;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function assertCompressedTensorsW4A16Group(group) {
|
|
419
|
+
const missing = [];
|
|
420
|
+
if (!group.packed) missing.push('weight_packed');
|
|
421
|
+
if (!group.scale) missing.push('weight_scale');
|
|
422
|
+
if (!group.shape) missing.push('weight_shape');
|
|
423
|
+
if (missing.length > 0) {
|
|
424
|
+
throw new Error(
|
|
425
|
+
`Compressed-tensors W4A16 tensor "${group.baseName}.weight" is missing companion tensors: ${missing.join(', ')}.`
|
|
426
|
+
);
|
|
427
|
+
}
|
|
428
|
+
const packedShape = normalizePositiveIntegerShape(group.packed.shape, group.packed.name);
|
|
429
|
+
if (packedShape.length !== 2) {
|
|
430
|
+
throw new Error(
|
|
431
|
+
`Compressed-tensors W4A16 tensor "${group.packed.name}" must be 2D; got shape ${JSON.stringify(packedShape)}.`
|
|
432
|
+
);
|
|
433
|
+
}
|
|
434
|
+
const scaleShape = normalizePositiveIntegerShape(group.scale.shape, group.scale.name);
|
|
435
|
+
if (scaleShape.length !== 2) {
|
|
436
|
+
throw new Error(
|
|
437
|
+
`Compressed-tensors W4A16 scale tensor "${group.scale.name}" must be 2D; got shape ${JSON.stringify(scaleShape)}.`
|
|
438
|
+
);
|
|
439
|
+
}
|
|
440
|
+
normalizePositiveIntegerShape(group.shape.shape, group.shape.name);
|
|
441
|
+
const shapeDtype = String(group.shape.dtype || '').toUpperCase();
|
|
442
|
+
if (shapeDtype !== 'I64' && shapeDtype !== 'I32' && shapeDtype !== 'U32') {
|
|
443
|
+
throw new Error(
|
|
444
|
+
`Compressed-tensors W4A16 shape tensor "${group.shape.name}" must use I64, I32, or U32; got "${group.shape.dtype}".`
|
|
445
|
+
);
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
const COMPRESSED_TENSORS_W4A16_PACKED_VALUES_PER_ELEMENT = {
|
|
450
|
+
U8: 2,
|
|
451
|
+
I8: 2,
|
|
452
|
+
U16: 4,
|
|
453
|
+
I16: 4,
|
|
454
|
+
U32: 8,
|
|
455
|
+
I32: 8,
|
|
456
|
+
};
|
|
457
|
+
|
|
458
|
+
function inferCompressedTensorsW4A16LogicalShape(group) {
|
|
459
|
+
const packedShape = normalizePositiveIntegerShape(group.packed.shape, group.packed.name);
|
|
460
|
+
const packedDtype = String(group.packed.dtype || '').toUpperCase();
|
|
461
|
+
const valuesPerElement = COMPRESSED_TENSORS_W4A16_PACKED_VALUES_PER_ELEMENT[packedDtype];
|
|
462
|
+
if (!valuesPerElement) {
|
|
463
|
+
throw new Error(
|
|
464
|
+
`Compressed-tensors W4A16 tensor "${group.packed.name}" has unsupported packed dtype "${group.packed.dtype}".`
|
|
465
|
+
);
|
|
466
|
+
}
|
|
467
|
+
return [packedShape[0], packedShape[1] * valuesPerElement];
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
function shouldNormalizeCompressedTensorsW4A16(converterConfig) {
|
|
471
|
+
return normalizeStorageQuant(converterConfig?.quantization?.weights) === 'w4a16'
|
|
472
|
+
|| normalizeStorageQuant(converterConfig?.quantization?.sourceQuantizationTarget) === 'w4a16';
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
function sortTensorsForConversion(tensors) {
|
|
476
|
+
return [...tensors].sort((left, right) => normalizeTensorName(left).localeCompare(normalizeTensorName(right)));
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
function normalizeCompressedTensorsW4A16(tensors, converterConfig) {
|
|
480
|
+
const groups = collectCompressedTensorsW4A16(tensors);
|
|
481
|
+
if (groups.size === 0) {
|
|
482
|
+
return tensors;
|
|
483
|
+
}
|
|
484
|
+
if (!shouldNormalizeCompressedTensorsW4A16(converterConfig)) {
|
|
485
|
+
throw new Error(
|
|
486
|
+
'Compressed-tensors W4A16 tensors were detected, but converter.quantization.weights or ' +
|
|
487
|
+
'converter.quantization.sourceQuantizationTarget is not "w4a16".'
|
|
488
|
+
);
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
const byName = new Map(tensors.map((tensor) => [normalizeTensorName(tensor), tensor]));
|
|
492
|
+
const consumed = new Set();
|
|
493
|
+
const companionByName = new Map();
|
|
494
|
+
const synthetic = [];
|
|
495
|
+
const sortedGroups = [...groups.values()].sort((left, right) => left.baseName.localeCompare(right.baseName));
|
|
496
|
+
for (const group of sortedGroups) {
|
|
497
|
+
assertCompressedTensorsW4A16Group(group);
|
|
498
|
+
const logicalName = `${group.baseName}.weight`;
|
|
499
|
+
if (byName.has(logicalName)) {
|
|
500
|
+
throw new Error(
|
|
501
|
+
`Compressed-tensors W4A16 logical tensor "${logicalName}" conflicts with an existing source tensor.`
|
|
502
|
+
);
|
|
503
|
+
}
|
|
504
|
+
consumed.add(group.packed.name);
|
|
505
|
+
companionByName.set(group.scale.name, {
|
|
506
|
+
...group.scale,
|
|
507
|
+
compressedTensorsW4A16Companion: {
|
|
508
|
+
role: 'scales',
|
|
509
|
+
primary: logicalName,
|
|
510
|
+
},
|
|
511
|
+
});
|
|
512
|
+
companionByName.set(group.shape.name, {
|
|
513
|
+
...group.shape,
|
|
514
|
+
compressedTensorsW4A16Companion: {
|
|
515
|
+
role: 'shape',
|
|
516
|
+
primary: logicalName,
|
|
517
|
+
},
|
|
518
|
+
});
|
|
519
|
+
synthetic.push({
|
|
520
|
+
...group.packed,
|
|
521
|
+
name: logicalName,
|
|
522
|
+
dtype: 'W4A16',
|
|
523
|
+
shape: inferCompressedTensorsW4A16LogicalShape(group),
|
|
524
|
+
packedSourceName: group.packed.name,
|
|
525
|
+
compressedTensorsW4A16: {
|
|
526
|
+
packed: group.packed.name,
|
|
527
|
+
scales: group.scale.name,
|
|
528
|
+
shape: group.shape.name,
|
|
529
|
+
},
|
|
530
|
+
storage: {
|
|
531
|
+
packing: 'w4a16',
|
|
532
|
+
blockShape: [32],
|
|
533
|
+
blockBytes: 16,
|
|
534
|
+
companions: [
|
|
535
|
+
{ role: 'scales', tensorId: group.scale.name },
|
|
536
|
+
{ role: 'shape', tensorId: group.shape.name },
|
|
537
|
+
],
|
|
538
|
+
},
|
|
539
|
+
});
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
return sortTensorsForConversion([
|
|
543
|
+
...tensors
|
|
544
|
+
.filter((tensor) => !consumed.has(normalizeTensorName(tensor)))
|
|
545
|
+
.map((tensor) => companionByName.get(normalizeTensorName(tensor)) ?? tensor),
|
|
546
|
+
...synthetic,
|
|
547
|
+
]);
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
function isCompressedTensorsW4A16CompanionTensor(tensor) {
|
|
551
|
+
return Boolean(
|
|
552
|
+
tensor?.compressedTensorsW4A16Companion
|
|
553
|
+
&& typeof tensor.compressedTensorsW4A16Companion === 'object'
|
|
554
|
+
);
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
function shouldMaterializeTiedLmHead(tensors, options) {
|
|
558
|
+
if (options?.inference?.output?.tieWordEmbeddings !== true) {
|
|
559
|
+
return false;
|
|
560
|
+
}
|
|
561
|
+
const lmHeadQuant = normalizeStorageQuant(options?.quantizationInfo?.lmHead ?? null);
|
|
562
|
+
if (!SOURCE_PACKED_QUANT_DTYPES.has(lmHeadQuant) && lmHeadQuant !== 'q4k') {
|
|
563
|
+
return false;
|
|
564
|
+
}
|
|
565
|
+
const embeddingQuant = normalizeStorageQuant(options?.quantizationInfo?.embeddings ?? null);
|
|
566
|
+
if (embeddingQuant === lmHeadQuant) {
|
|
567
|
+
return false;
|
|
568
|
+
}
|
|
569
|
+
return !tensors.some((tensor) => resolveTensorRole(tensor) === 'lm_head');
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
function resolveTiedLmHeadName(embeddingName) {
|
|
573
|
+
const name = typeof embeddingName === 'string' ? embeddingName.trim() : '';
|
|
574
|
+
if (name === 'model.language_model.model.embed_tokens.weight') {
|
|
575
|
+
return 'model.language_model.lm_head.weight';
|
|
576
|
+
}
|
|
577
|
+
if (name === 'language_model.model.embed_tokens.weight') {
|
|
578
|
+
return 'language_model.lm_head.weight';
|
|
579
|
+
}
|
|
580
|
+
if (name === 'model.language_model.embed_tokens.weight') {
|
|
581
|
+
return 'model.language_model.lm_head.weight';
|
|
582
|
+
}
|
|
583
|
+
if (name === 'model.decoder.embed_tokens.weight') {
|
|
584
|
+
return 'lm_head.weight';
|
|
585
|
+
}
|
|
586
|
+
if (name === 'decoder.embed_tokens.weight') {
|
|
587
|
+
return 'lm_head.weight';
|
|
588
|
+
}
|
|
589
|
+
if (name === 'model.encoder.language_model.embed_tokens.weight') {
|
|
590
|
+
return 'lm_head.weight';
|
|
591
|
+
}
|
|
592
|
+
if (name === 'language_model.embed_tokens.weight') {
|
|
593
|
+
return 'language_model.lm_head.weight';
|
|
594
|
+
}
|
|
595
|
+
if (name === 'model.embed_tokens.weight' || name === 'embed_tokens.weight') {
|
|
596
|
+
return 'lm_head.weight';
|
|
597
|
+
}
|
|
598
|
+
return 'lm_head.weight';
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
function resolveTiedEmbeddingTensor(tensors, modelType) {
|
|
602
|
+
const candidates = tensors.filter((tensor) => (
|
|
603
|
+
resolveTensorRole(tensor) === 'embedding'
|
|
604
|
+
&& resolveTensorGroup(tensor, modelType) === 'embed'
|
|
605
|
+
&& Array.isArray(tensor?.shape)
|
|
606
|
+
&& tensor.shape.length === 2
|
|
607
|
+
));
|
|
608
|
+
return candidates[0] ?? null;
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
function materializeTiedLmHeadTensor(tensors, options) {
|
|
612
|
+
if (!shouldMaterializeTiedLmHead(tensors, options)) {
|
|
613
|
+
return tensors;
|
|
614
|
+
}
|
|
615
|
+
const embedding = resolveTiedEmbeddingTensor(tensors, options?.modelType ?? 'transformer');
|
|
616
|
+
if (!embedding) {
|
|
617
|
+
throw new Error(
|
|
618
|
+
'Cannot materialize tied Q4K LM head: no 2D token embedding tensor was selected. '
|
|
619
|
+
+ 'Check inference.output.tieWordEmbeddings and the conversion tensor filter.'
|
|
620
|
+
);
|
|
621
|
+
}
|
|
622
|
+
const name = resolveTiedLmHeadName(embedding.name);
|
|
623
|
+
if (tensors.some((tensor) => tensor?.name === name)) {
|
|
624
|
+
throw new Error(`Cannot materialize tied Q4K LM head: synthetic tensor name "${name}" already exists.`);
|
|
625
|
+
}
|
|
626
|
+
return [
|
|
627
|
+
...tensors,
|
|
628
|
+
{
|
|
629
|
+
...embedding,
|
|
630
|
+
name,
|
|
631
|
+
role: 'lm_head',
|
|
632
|
+
group: 'head',
|
|
633
|
+
sourceTensorName: embedding.name,
|
|
634
|
+
},
|
|
635
|
+
];
|
|
636
|
+
}
|
|
637
|
+
|
|
267
638
|
function toFloat32ForQ4K(tensorData, sourceDtype, tensorName) {
|
|
268
639
|
const dtype = String(sourceDtype || '').toUpperCase();
|
|
269
640
|
if (dtype === 'F32') {
|
|
@@ -454,6 +825,9 @@ function resolveMoEExpertFormat(rawConfig, resolvedModelType, quantizationInfo,
|
|
|
454
825
|
if (modelType.includes('gpt_oss') || modelType.includes('gpt-oss') || modelType.includes('gptoss')) {
|
|
455
826
|
return 'gpt-oss';
|
|
456
827
|
}
|
|
828
|
+
if (modelType === 'diffusion_gemma' || modelType === 'diffusion_gemma_text') {
|
|
829
|
+
return 'gemma4';
|
|
830
|
+
}
|
|
457
831
|
return 'mixtral';
|
|
458
832
|
}
|
|
459
833
|
|
|
@@ -462,7 +836,10 @@ function normalizeMoEConfig(config, contextLabel) {
|
|
|
462
836
|
const numExperts = Number(config.numExperts);
|
|
463
837
|
const numExpertsPerToken = Number(config.numExpertsPerToken);
|
|
464
838
|
const expertFormat = String(config.expertFormat || '').trim();
|
|
465
|
-
const
|
|
839
|
+
const expertIntermediateSize = config.expertIntermediateSize == null
|
|
840
|
+
? null
|
|
841
|
+
: Number(config.expertIntermediateSize);
|
|
842
|
+
const allowedExpertFormat = expertFormat === 'gpt-oss' || expertFormat === 'mixtral' || expertFormat === 'gemma4';
|
|
466
843
|
if (!Number.isFinite(numExperts) || numExperts <= 0) {
|
|
467
844
|
throw new Error(`Invalid moeConfig.numExperts for ${contextLabel}`);
|
|
468
845
|
}
|
|
@@ -475,10 +852,20 @@ function normalizeMoEConfig(config, contextLabel) {
|
|
|
475
852
|
if (!allowedExpertFormat) {
|
|
476
853
|
throw new Error(`Invalid moeConfig.expertFormat for ${contextLabel}: "${expertFormat}"`);
|
|
477
854
|
}
|
|
855
|
+
if (
|
|
856
|
+
expertIntermediateSize != null
|
|
857
|
+
&& (!Number.isFinite(expertIntermediateSize) || expertIntermediateSize <= 0)
|
|
858
|
+
) {
|
|
859
|
+
throw new Error(`Invalid moeConfig.expertIntermediateSize for ${contextLabel}`);
|
|
860
|
+
}
|
|
861
|
+
if (expertFormat === 'gemma4' && expertIntermediateSize == null) {
|
|
862
|
+
throw new Error(`Invalid moeConfig for ${contextLabel}: gemma4 experts require expertIntermediateSize`);
|
|
863
|
+
}
|
|
478
864
|
return {
|
|
479
865
|
numExperts,
|
|
480
866
|
numExpertsPerToken,
|
|
481
867
|
expertFormat,
|
|
868
|
+
...(expertIntermediateSize == null ? {} : { expertIntermediateSize }),
|
|
482
869
|
};
|
|
483
870
|
}
|
|
484
871
|
|
|
@@ -523,9 +910,17 @@ export function resolveManifestMoEConfig(model, options, rawConfig, resolvedMode
|
|
|
523
910
|
options?.quantizationInfo ?? null,
|
|
524
911
|
null
|
|
525
912
|
);
|
|
913
|
+
const expertIntermediateSize = expertFormat === 'gemma4'
|
|
914
|
+
? resolveMoEConfigNumber(rawConfig, 'moe_intermediate_size', 'expert_intermediate_size')
|
|
915
|
+
: null;
|
|
526
916
|
|
|
527
917
|
return normalizeMoEConfig(
|
|
528
|
-
{
|
|
918
|
+
{
|
|
919
|
+
numExperts,
|
|
920
|
+
numExpertsPerToken,
|
|
921
|
+
expertFormat,
|
|
922
|
+
...(expertIntermediateSize == null ? {} : { expertIntermediateSize }),
|
|
923
|
+
},
|
|
529
924
|
options?.modelId ?? 'model'
|
|
530
925
|
);
|
|
531
926
|
}
|
|
@@ -722,6 +1117,143 @@ export function shouldQuantize(tensorName, shape, options = {}) {
|
|
|
722
1117
|
return true;
|
|
723
1118
|
}
|
|
724
1119
|
|
|
1120
|
+
const GEMMA4_PLE_TENSOR_SUFFIXES = [
|
|
1121
|
+
'embed_tokens_per_layer.weight',
|
|
1122
|
+
'per_layer_embeddings.weight',
|
|
1123
|
+
// GGUF (unsloth / ggml-org) naming for Gemma 4's PLE table.
|
|
1124
|
+
'per_layer_token_embd.weight',
|
|
1125
|
+
];
|
|
1126
|
+
|
|
1127
|
+
export function isGemma4PerLayerEmbedTensor(tensorName) {
|
|
1128
|
+
if (typeof tensorName !== 'string') return false;
|
|
1129
|
+
const normalized = tensorName.trim().toLowerCase();
|
|
1130
|
+
return GEMMA4_PLE_TENSOR_SUFFIXES.some((suffix) => normalized.endsWith(suffix));
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
function validateInt4PleMaterializationContract(tensorLocations, inference, modelId) {
|
|
1134
|
+
const materialization = inference?.session?.perLayerInputs?.materialization;
|
|
1135
|
+
if (materialization !== 'gpu_split_tables') {
|
|
1136
|
+
return;
|
|
1137
|
+
}
|
|
1138
|
+
for (const [name, location] of Object.entries(tensorLocations ?? {})) {
|
|
1139
|
+
const sourceTransform = location?.sourceTransform ?? null;
|
|
1140
|
+
if (
|
|
1141
|
+
isGemma4PerLayerEmbedTensor(name)
|
|
1142
|
+
&& sourceTransform?.kind === 'litert_axis_dequant'
|
|
1143
|
+
&& String(sourceTransform?.sourceDtype ?? '').toUpperCase() === 'INT4'
|
|
1144
|
+
) {
|
|
1145
|
+
throw new Error(
|
|
1146
|
+
`Manifest "${modelId}" Gemma 4 INT4 PLE tensor "${name}" cannot use ` +
|
|
1147
|
+
'inference.session.perLayerInputs.materialization="gpu_split_tables". ' +
|
|
1148
|
+
'Use materialization="range_backed" or disable INT4 PLE quantization.'
|
|
1149
|
+
);
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
function resolveOriginalTensorShape(options) {
|
|
1155
|
+
const shape = options?.originalTensorShape;
|
|
1156
|
+
if (!Array.isArray(shape) || shape.length !== 2) {
|
|
1157
|
+
return null;
|
|
1158
|
+
}
|
|
1159
|
+
const rows = Number(shape[0]);
|
|
1160
|
+
const cols = Number(shape[1]);
|
|
1161
|
+
if (!Number.isInteger(rows) || rows <= 0 || !Number.isInteger(cols) || cols <= 0) {
|
|
1162
|
+
return null;
|
|
1163
|
+
}
|
|
1164
|
+
return [rows, cols];
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
function resolvePerLayerEmbeddingQuant(options) {
|
|
1168
|
+
const value = (
|
|
1169
|
+
options?.perLayerEmbeddings
|
|
1170
|
+
?? options?.quantizationInfo?.perLayerEmbeddings
|
|
1171
|
+
?? null
|
|
1172
|
+
);
|
|
1173
|
+
if (value == null) return null;
|
|
1174
|
+
return String(value).trim().toLowerCase().replace(/\s+/g, '_') || null;
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
function canInt4QuantizePerRow(tensor, options) {
|
|
1178
|
+
if (options?.skipInt4PlePerRow === true) return false;
|
|
1179
|
+
if (resolvePerLayerEmbeddingQuant(options) !== 'int4_per_row') return false;
|
|
1180
|
+
if (!Array.isArray(tensor.shape) || tensor.shape.length !== 2) return false;
|
|
1181
|
+
const [rows, cols] = tensor.shape;
|
|
1182
|
+
if (!Number.isInteger(rows) || rows <= 0) return false;
|
|
1183
|
+
if (!Number.isInteger(cols) || cols <= 0) return false;
|
|
1184
|
+
if ((cols & 1) !== 0) return false;
|
|
1185
|
+
const originalShape = resolveOriginalTensorShape(options);
|
|
1186
|
+
if (originalShape && originalShape[1] !== cols) return false;
|
|
1187
|
+
// Per-row symmetric INT4 produces one scale per row. PLE semantics require
|
|
1188
|
+
// one scale per VOCAB token (typically 262144 for Gemma 4). HF stores PLE
|
|
1189
|
+
// as [vocab, hidden] (rows = vocab ≫ cols). GGUF stores it transposed as
|
|
1190
|
+
// [hidden, vocab] (rows = hidden ≪ cols). Quantizing the GGUF layout with
|
|
1191
|
+
// per-row would wrongly share a single scale across all vocab tokens.
|
|
1192
|
+
// Require the tensor to already be in [vocab, hidden] layout. Callers that
|
|
1193
|
+
// have a GGUF PLE must transpose before reaching this function.
|
|
1194
|
+
if (rows <= cols && (!originalShape || originalShape[0] <= originalShape[1])) return false;
|
|
1195
|
+
const srcDtype = String(tensor.dtype || '').toUpperCase();
|
|
1196
|
+
return srcDtype === 'F32' || srcDtype === 'F16' || srcDtype === 'BF16';
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
function toFloat32FromTensor(bytes, sourceDtype, tensorName) {
|
|
1200
|
+
const src = String(sourceDtype || '').toUpperCase();
|
|
1201
|
+
if (src === 'F32') {
|
|
1202
|
+
if (bytes.byteLength % 4 !== 0) {
|
|
1203
|
+
throw new Error(`Invalid F32 tensor byte length for ${tensorName}: ${bytes.byteLength}`);
|
|
1204
|
+
}
|
|
1205
|
+
return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
|
|
1206
|
+
}
|
|
1207
|
+
if (src === 'F16') {
|
|
1208
|
+
if (bytes.byteLength % 2 !== 0) {
|
|
1209
|
+
throw new Error(`Invalid F16 tensor byte length for ${tensorName}: ${bytes.byteLength}`);
|
|
1210
|
+
}
|
|
1211
|
+
const src16 = new Uint16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2);
|
|
1212
|
+
const out = new Float32Array(src16.length);
|
|
1213
|
+
for (let i = 0; i < src16.length; i++) out[i] = float16ToFloat32(src16[i]);
|
|
1214
|
+
return out;
|
|
1215
|
+
}
|
|
1216
|
+
if (src === 'BF16') {
|
|
1217
|
+
if (bytes.byteLength % 2 !== 0) {
|
|
1218
|
+
throw new Error(`Invalid BF16 tensor byte length for ${tensorName}: ${bytes.byteLength}`);
|
|
1219
|
+
}
|
|
1220
|
+
const src16 = new Uint16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2);
|
|
1221
|
+
const out = new Float32Array(src16.length);
|
|
1222
|
+
for (let i = 0; i < src16.length; i++) out[i] = bf16ToFloat32(src16[i]);
|
|
1223
|
+
return out;
|
|
1224
|
+
}
|
|
1225
|
+
throw new Error(`Unsupported source dtype "${sourceDtype}" for PLE INT4 quantization of ${tensorName}`);
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
function buildInt4PerRowPleTransform(tensor, bytes, sourceDtype, options = {}) {
|
|
1229
|
+
const f32 = toFloat32FromTensor(bytes, sourceDtype, tensor.name);
|
|
1230
|
+
const { quantized, scales } = quantizeToInt4PerRowSymmetric(f32, tensor.shape);
|
|
1231
|
+
const scalesBytes = new Uint8Array(scales.buffer, scales.byteOffset, scales.byteLength);
|
|
1232
|
+
const [rows, cols] = resolveOriginalTensorShape(options) ?? tensor.shape;
|
|
1233
|
+
return {
|
|
1234
|
+
tensorData: quantized,
|
|
1235
|
+
companionData: scalesBytes,
|
|
1236
|
+
// outDtype is the LOGICAL dtype the tensor resolves to after dequant at
|
|
1237
|
+
// load time. Storage dtype (INT4) is carried in sourceTransform.sourceDtype.
|
|
1238
|
+
outDtype: 'F16',
|
|
1239
|
+
outLayout: 'row',
|
|
1240
|
+
sourceDtype: String(sourceDtype || '').toUpperCase(),
|
|
1241
|
+
tensorTargetQuant: 'int4_per_row_ple',
|
|
1242
|
+
sourceTransform: {
|
|
1243
|
+
kind: 'litert_axis_dequant',
|
|
1244
|
+
scheme: 'per_axis_affine',
|
|
1245
|
+
sourceDtype: 'INT4',
|
|
1246
|
+
targetDtype: 'F16',
|
|
1247
|
+
storageEncoding: 'offset_binary',
|
|
1248
|
+
scaleSemantics: 'step',
|
|
1249
|
+
storageShape: [rows, cols],
|
|
1250
|
+
quantAxis: 1,
|
|
1251
|
+
// scaleSource gets filled in by the writer with the scales companion's
|
|
1252
|
+
// shard/offset/size after appendTensorBytes().
|
|
1253
|
+
},
|
|
1254
|
+
};
|
|
1255
|
+
}
|
|
1256
|
+
|
|
725
1257
|
export function transformTensorBytes(tensor, rawData, options = {}) {
|
|
726
1258
|
const tensorDataInput = rawData instanceof Uint8Array ? rawData : new Uint8Array(rawData);
|
|
727
1259
|
let tensorData = tensorDataInput;
|
|
@@ -750,11 +1282,83 @@ export function transformTensorBytes(tensor, rawData, options = {}) {
|
|
|
750
1282
|
: null
|
|
751
1283
|
);
|
|
752
1284
|
|
|
1285
|
+
// Gemma 4 per-layer embeddings use MediaPipe's INT4 per-row symmetric
|
|
1286
|
+
// quantization (verified against gemma-4-E2B-it.litertlm composites:
|
|
1287
|
+
// quantizedDimension=0, one F32 scale per vocab row, zero_point=0). Saves
|
|
1288
|
+
// ~3.5 GB per model vs the default F16 path. Runtime reads via
|
|
1289
|
+
// sourceTransform.kind=litert_axis_dequant with scaleSemantics=step.
|
|
1290
|
+
if (isGemma4PerLayerEmbedTensor(tensor.name) && canInt4QuantizePerRow(tensor, options)) {
|
|
1291
|
+
return buildInt4PerRowPleTransform(tensor, tensorDataInput, sourceDtype, options);
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
if (isCompressedTensorsW4A16CompanionTensor(tensor)) {
|
|
1295
|
+
return {
|
|
1296
|
+
tensorData,
|
|
1297
|
+
outDtype: sourceDtype,
|
|
1298
|
+
outLayout: null,
|
|
1299
|
+
sourceDtype,
|
|
1300
|
+
tensorTargetQuant: null,
|
|
1301
|
+
};
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1304
|
+
if (SOURCE_PACKED_QUANT_DTYPES.has(tensorTargetQuant)) {
|
|
1305
|
+
const sourceQuant = normalizeStorageQuant(sourceDtype);
|
|
1306
|
+
if (sourceQuant === tensorTargetQuant) {
|
|
1307
|
+
const descriptor = cloneSourcePackedStorageDescriptor(tensorTargetQuant);
|
|
1308
|
+
const sourceCompanions = Array.isArray(tensor?.storage?.companions)
|
|
1309
|
+
? tensor.storage.companions.map((companion) => ({ ...companion }))
|
|
1310
|
+
: null;
|
|
1311
|
+
return {
|
|
1312
|
+
tensorData,
|
|
1313
|
+
outDtype: SOURCE_PACKED_MANIFEST_DTYPES[tensorTargetQuant],
|
|
1314
|
+
outLayout: null,
|
|
1315
|
+
sourceDtype,
|
|
1316
|
+
tensorTargetQuant,
|
|
1317
|
+
storage: {
|
|
1318
|
+
...descriptor,
|
|
1319
|
+
...(sourceCompanions ? { companions: sourceCompanions } : {}),
|
|
1320
|
+
},
|
|
1321
|
+
};
|
|
1322
|
+
}
|
|
1323
|
+
const roleQuant = resolveExplicitRoleQuant(tensor, quantizationInfo);
|
|
1324
|
+
const requiresPackedSource = (
|
|
1325
|
+
forceQuantizeDecision
|
|
1326
|
+
?? (
|
|
1327
|
+
roleQuant === tensorTargetQuant
|
|
1328
|
+
|| shouldQuantize(tensor.name, tensor.shape, {
|
|
1329
|
+
quantizeEmbeddings,
|
|
1330
|
+
modulesToNotConvert,
|
|
1331
|
+
role: tensor.role ?? null,
|
|
1332
|
+
})
|
|
1333
|
+
)
|
|
1334
|
+
);
|
|
1335
|
+
if (!requiresPackedSource) {
|
|
1336
|
+
return {
|
|
1337
|
+
tensorData,
|
|
1338
|
+
outDtype: sourceDtype,
|
|
1339
|
+
outLayout: null,
|
|
1340
|
+
sourceDtype,
|
|
1341
|
+
tensorTargetQuant,
|
|
1342
|
+
};
|
|
1343
|
+
}
|
|
1344
|
+
throw new Error(
|
|
1345
|
+
`Cannot materialize ${tensorTargetQuant} for ${tensor.name}: ` +
|
|
1346
|
+
`native import requires source dtype ${SOURCE_PACKED_MANIFEST_DTYPES[tensorTargetQuant]}; ` +
|
|
1347
|
+
'the converter does not re-quantize tensors into this packed format.'
|
|
1348
|
+
);
|
|
1349
|
+
}
|
|
1350
|
+
|
|
753
1351
|
if (tensorTargetQuant === 'q4k') {
|
|
754
1352
|
const sourceQuant = normalizeStorageQuant(sourceDtype);
|
|
1353
|
+
const tensorRole = resolveTensorRole(tensor);
|
|
1354
|
+
const isMatrixLikeShape = Array.isArray(tensor.shape) && tensor.shape.length >= 2;
|
|
1355
|
+
const is2DMatrixShape = Array.isArray(tensor.shape) && tensor.shape.length === 2;
|
|
1356
|
+
const useQ4KRowWise = isMatrixLikeShape
|
|
1357
|
+
&& q4kLayout === 'row'
|
|
1358
|
+
&& (is2DMatrixShape || tensorRole === 'expert');
|
|
755
1359
|
if (sourceQuant === 'q4k') {
|
|
756
1360
|
outDtype = 'Q4_K_M';
|
|
757
|
-
if (
|
|
1361
|
+
if (is2DMatrixShape) {
|
|
758
1362
|
outLayout = q4kLayout;
|
|
759
1363
|
}
|
|
760
1364
|
return {
|
|
@@ -776,15 +1380,17 @@ export function transformTensorBytes(tensor, rawData, options = {}) {
|
|
|
776
1380
|
if (shouldQuantizeTensor) {
|
|
777
1381
|
const f32Data = toFloat32ForQ4K(tensorData, sourceDtype, tensor.name);
|
|
778
1382
|
const quantized = (
|
|
779
|
-
|
|
1383
|
+
is2DMatrixShape
|
|
780
1384
|
? (q4kLayout === 'col'
|
|
781
1385
|
? quantizeToQ4KMColumnWise(f32Data, tensor.shape)
|
|
782
1386
|
: quantizeToQ4KMRowWise(f32Data, tensor.shape))
|
|
1387
|
+
: useQ4KRowWise
|
|
1388
|
+
? quantizeToQ4KMRowWise(f32Data, tensor.shape)
|
|
783
1389
|
: quantizeToQ4KM(f32Data, tensor.shape)
|
|
784
1390
|
);
|
|
785
1391
|
tensorData = quantized.quantized;
|
|
786
1392
|
outDtype = 'Q4_K_M';
|
|
787
|
-
if (
|
|
1393
|
+
if (is2DMatrixShape || useQ4KRowWise) {
|
|
788
1394
|
outLayout = q4kLayout;
|
|
789
1395
|
}
|
|
790
1396
|
} else if (sourceDtype === 'BF16') {
|
|
@@ -963,6 +1569,7 @@ export function extractArchitecture(config, ggufConfig) {
|
|
|
963
1569
|
'num_attention_heads'
|
|
964
1570
|
);
|
|
965
1571
|
const numKVHeads = fromConfig('num_key_value_heads', 'num_kv_heads') ?? numHeads;
|
|
1572
|
+
const numGlobalKVHeads = fromConfig('num_global_key_value_heads', 'num_global_kv_heads');
|
|
966
1573
|
const headDimFromConfig = fromConfig('head_dim') ?? Math.floor(hiddenSize / numHeads);
|
|
967
1574
|
const vocabSize = requireNumber(
|
|
968
1575
|
fromConfig('vocab_size', 'n_vocab'),
|
|
@@ -1006,6 +1613,7 @@ export function extractArchitecture(config, ggufConfig) {
|
|
|
1006
1613
|
intermediateSize,
|
|
1007
1614
|
numAttentionHeads: numHeads,
|
|
1008
1615
|
numKeyValueHeads: numKVHeads,
|
|
1616
|
+
numGlobalKeyValueHeads: numGlobalKVHeads ?? undefined,
|
|
1009
1617
|
headDim: headDimFromConfig,
|
|
1010
1618
|
vocabSize,
|
|
1011
1619
|
maxSeqLen,
|
|
@@ -1052,15 +1660,35 @@ export function extractArchitecture(config, ggufConfig) {
|
|
|
1052
1660
|
'contextLength'
|
|
1053
1661
|
);
|
|
1054
1662
|
|
|
1663
|
+
// Gemma 4-specific fields (optional — undefined on non-Gemma-4 GGUFs).
|
|
1664
|
+
// key_length is per-head; pick the larger of (key_length, key_length_swa)
|
|
1665
|
+
// for globalHeadDim and the smaller for headDim. Matches the mixed-geometry
|
|
1666
|
+
// KV cache expected by src/inference/pipelines/text/layer.js.
|
|
1667
|
+
const keyLen = firstNumber(c.attentionKeyLength);
|
|
1668
|
+
const keyLenSwa = firstNumber(c.attentionKeyLengthSwa);
|
|
1669
|
+
const headDimFromGguf = keyLenSwa != null && keyLen != null
|
|
1670
|
+
? Math.min(keyLen, keyLenSwa)
|
|
1671
|
+
: (keyLenSwa ?? Math.floor(hiddenSize / numHeads));
|
|
1672
|
+
const globalHeadDim = (keyLen != null && keyLenSwa != null && keyLen !== keyLenSwa)
|
|
1673
|
+
? Math.max(keyLen, keyLenSwa)
|
|
1674
|
+
: undefined;
|
|
1675
|
+
const numKvSharedLayers = firstNumber(c.numKvSharedLayers);
|
|
1676
|
+
const hiddenSizePerLayerInput = firstNumber(c.hiddenSizePerLayerInput);
|
|
1677
|
+
const vocabSizePerLayerInput = hiddenSizePerLayerInput != null ? vocabSize : undefined;
|
|
1678
|
+
|
|
1055
1679
|
return {
|
|
1056
1680
|
numLayers,
|
|
1057
1681
|
hiddenSize,
|
|
1058
1682
|
intermediateSize,
|
|
1059
1683
|
numAttentionHeads: numHeads,
|
|
1060
1684
|
numKeyValueHeads: numKVHeads,
|
|
1061
|
-
headDim:
|
|
1685
|
+
headDim: headDimFromGguf,
|
|
1062
1686
|
vocabSize,
|
|
1063
1687
|
maxSeqLen,
|
|
1688
|
+
...(globalHeadDim != null ? { globalHeadDim } : {}),
|
|
1689
|
+
...(numKvSharedLayers != null ? { numKvSharedLayers } : {}),
|
|
1690
|
+
...(hiddenSizePerLayerInput != null ? { hiddenSizePerLayerInput } : {}),
|
|
1691
|
+
...(vocabSizePerLayerInput != null ? { vocabSizePerLayerInput } : {}),
|
|
1064
1692
|
};
|
|
1065
1693
|
}
|
|
1066
1694
|
|
|
@@ -1083,12 +1711,37 @@ function getNestedTextConfig(config) {
|
|
|
1083
1711
|
function resolveGemma4TextConfig(rawConfig) {
|
|
1084
1712
|
const textConfig = getNestedTextConfig(rawConfig);
|
|
1085
1713
|
const modelType = String(textConfig?.model_type ?? rawConfig?.model_type ?? '').trim().toLowerCase();
|
|
1086
|
-
if (
|
|
1714
|
+
if (
|
|
1715
|
+
modelType !== 'gemma4'
|
|
1716
|
+
&& modelType !== 'gemma4_text'
|
|
1717
|
+
&& modelType !== 'gemma4_unified'
|
|
1718
|
+
&& modelType !== 'gemma4_unified_text'
|
|
1719
|
+
) {
|
|
1087
1720
|
return null;
|
|
1088
1721
|
}
|
|
1089
1722
|
return textConfig ?? rawConfig ?? null;
|
|
1090
1723
|
}
|
|
1091
1724
|
|
|
1725
|
+
function resolveDiffusionGemmaConfig(rawConfig, resolvedModelType = null) {
|
|
1726
|
+
const textConfig = getNestedTextConfig(rawConfig);
|
|
1727
|
+
const modelType = String(
|
|
1728
|
+
resolvedModelType
|
|
1729
|
+
?? rawConfig?.model_type
|
|
1730
|
+
?? textConfig?.model_type
|
|
1731
|
+
?? ''
|
|
1732
|
+
).trim().toLowerCase();
|
|
1733
|
+
const textModelType = String(textConfig?.model_type ?? '').trim().toLowerCase();
|
|
1734
|
+
if (
|
|
1735
|
+
modelType !== 'diffusion_gemma'
|
|
1736
|
+
&& modelType !== 'diffusion_gemma_text'
|
|
1737
|
+
&& textModelType !== 'diffusion_gemma'
|
|
1738
|
+
&& textModelType !== 'diffusion_gemma_text'
|
|
1739
|
+
) {
|
|
1740
|
+
return null;
|
|
1741
|
+
}
|
|
1742
|
+
return textConfig ?? rawConfig ?? {};
|
|
1743
|
+
}
|
|
1744
|
+
|
|
1092
1745
|
function collectGemma4UnsupportedTensorFlags(tensors) {
|
|
1093
1746
|
const names = Array.isArray(tensors) ? tensors.map((tensor) => normalizeTensorName(tensor).toLowerCase()) : [];
|
|
1094
1747
|
const flags = [];
|
|
@@ -1160,6 +1813,143 @@ function assertSupportedGemma4Conversion(model, tensors, modelId) {
|
|
|
1160
1813
|
);
|
|
1161
1814
|
}
|
|
1162
1815
|
|
|
1816
|
+
function readDiffusionGemmaPositiveInteger(value, label, modelId) {
|
|
1817
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
1818
|
+
throw new Error(`DiffusionGemma model "${modelId}" requires ${label} to be a positive integer.`);
|
|
1819
|
+
}
|
|
1820
|
+
return value;
|
|
1821
|
+
}
|
|
1822
|
+
|
|
1823
|
+
function readDiffusionGemmaNonNegativeNumber(value, label, modelId) {
|
|
1824
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
|
1825
|
+
throw new Error(`DiffusionGemma model "${modelId}" requires ${label} to be a non-negative finite number.`);
|
|
1826
|
+
}
|
|
1827
|
+
return value;
|
|
1828
|
+
}
|
|
1829
|
+
|
|
1830
|
+
function readDiffusionGemmaEntropyBound(generationConfig, modelId) {
|
|
1831
|
+
if (generationConfig && Object.hasOwn(generationConfig, 'entropy_bound')) {
|
|
1832
|
+
return readDiffusionGemmaNonNegativeNumber(
|
|
1833
|
+
generationConfig.entropy_bound,
|
|
1834
|
+
'generation_config.entropy_bound',
|
|
1835
|
+
modelId
|
|
1836
|
+
);
|
|
1837
|
+
}
|
|
1838
|
+
return readDiffusionGemmaNonNegativeNumber(
|
|
1839
|
+
generationConfig?.sampler_config?.entropy_bound,
|
|
1840
|
+
'generation_config.sampler_config.entropy_bound',
|
|
1841
|
+
modelId
|
|
1842
|
+
);
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
function readDiffusionGemmaNullableTokenId(value, label, modelId) {
|
|
1846
|
+
if (value == null) return null;
|
|
1847
|
+
if (!Number.isInteger(value) || value < 0) {
|
|
1848
|
+
throw new Error(`DiffusionGemma model "${modelId}" requires ${label} to be null or a non-negative integer.`);
|
|
1849
|
+
}
|
|
1850
|
+
return value;
|
|
1851
|
+
}
|
|
1852
|
+
|
|
1853
|
+
function readDiffusionGemmaTokenId(value, label, modelId) {
|
|
1854
|
+
if (!Number.isInteger(value) || value < 0) {
|
|
1855
|
+
throw new Error(`DiffusionGemma model "${modelId}" requires ${label} to be a non-negative integer.`);
|
|
1856
|
+
}
|
|
1857
|
+
return value;
|
|
1858
|
+
}
|
|
1859
|
+
|
|
1860
|
+
function readDiffusionGemmaEosTokenIds(rawConfig, generationConfig, modelId) {
|
|
1861
|
+
const eos = resolveTokenizerIds(
|
|
1862
|
+
generationConfig?.eos_token_id
|
|
1863
|
+
?? generationConfig?.eos_token_ids
|
|
1864
|
+
?? rawConfig?.eos_token_id
|
|
1865
|
+
?? rawConfig?.eos_token_ids
|
|
1866
|
+
);
|
|
1867
|
+
if (!Array.isArray(eos) || eos.length === 0) {
|
|
1868
|
+
throw new Error(`DiffusionGemma model "${modelId}" requires generation_config.eos_token_id.`);
|
|
1869
|
+
}
|
|
1870
|
+
return eos;
|
|
1871
|
+
}
|
|
1872
|
+
|
|
1873
|
+
function resolveDiffusionGemmaInferenceContract(rawConfig, generationConfig, modelId) {
|
|
1874
|
+
const canvasLength = readDiffusionGemmaPositiveInteger(
|
|
1875
|
+
rawConfig?.canvas_length,
|
|
1876
|
+
'config.canvas_length',
|
|
1877
|
+
modelId
|
|
1878
|
+
);
|
|
1879
|
+
const maxDenoisingSteps = readDiffusionGemmaPositiveInteger(
|
|
1880
|
+
generationConfig?.max_denoising_steps,
|
|
1881
|
+
'generation_config.max_denoising_steps',
|
|
1882
|
+
modelId
|
|
1883
|
+
);
|
|
1884
|
+
const maxNewTokens = readDiffusionGemmaPositiveInteger(
|
|
1885
|
+
generationConfig?.max_new_tokens,
|
|
1886
|
+
'generation_config.max_new_tokens',
|
|
1887
|
+
modelId
|
|
1888
|
+
);
|
|
1889
|
+
const stabilityThreshold = readDiffusionGemmaPositiveInteger(
|
|
1890
|
+
generationConfig?.stability_threshold,
|
|
1891
|
+
'generation_config.stability_threshold',
|
|
1892
|
+
modelId
|
|
1893
|
+
);
|
|
1894
|
+
const padTokenId = readDiffusionGemmaTokenId(
|
|
1895
|
+
generationConfig?.pad_token_id ?? rawConfig?.pad_token_id,
|
|
1896
|
+
'generation_config.pad_token_id',
|
|
1897
|
+
modelId
|
|
1898
|
+
);
|
|
1899
|
+
const tMin = readDiffusionGemmaNonNegativeNumber(
|
|
1900
|
+
generationConfig?.t_min,
|
|
1901
|
+
'generation_config.t_min',
|
|
1902
|
+
modelId
|
|
1903
|
+
);
|
|
1904
|
+
const tMax = readDiffusionGemmaNonNegativeNumber(
|
|
1905
|
+
generationConfig?.t_max,
|
|
1906
|
+
'generation_config.t_max',
|
|
1907
|
+
modelId
|
|
1908
|
+
);
|
|
1909
|
+
if (tMax < tMin) {
|
|
1910
|
+
throw new Error(`DiffusionGemma model "${modelId}" requires generation_config.t_max >= t_min.`);
|
|
1911
|
+
}
|
|
1912
|
+
return {
|
|
1913
|
+
canvasLength,
|
|
1914
|
+
maxDenoisingSteps,
|
|
1915
|
+
maxNewTokens,
|
|
1916
|
+
tMin,
|
|
1917
|
+
tMax,
|
|
1918
|
+
entropyBound: readDiffusionGemmaEntropyBound(generationConfig, modelId),
|
|
1919
|
+
confidenceThreshold: readDiffusionGemmaNonNegativeNumber(
|
|
1920
|
+
generationConfig?.confidence_threshold,
|
|
1921
|
+
'generation_config.confidence_threshold',
|
|
1922
|
+
modelId
|
|
1923
|
+
),
|
|
1924
|
+
stabilityThreshold,
|
|
1925
|
+
padTokenId,
|
|
1926
|
+
eosTokenIds: readDiffusionGemmaEosTokenIds(rawConfig, generationConfig, modelId),
|
|
1927
|
+
boiTokenId: readDiffusionGemmaNullableTokenId(rawConfig?.boi_token_id, 'config.boi_token_id', modelId),
|
|
1928
|
+
eoiTokenId: readDiffusionGemmaNullableTokenId(rawConfig?.eoi_token_id, 'config.eoi_token_id', modelId),
|
|
1929
|
+
imageTokenId: readDiffusionGemmaNullableTokenId(rawConfig?.image_token_id, 'config.image_token_id', modelId),
|
|
1930
|
+
selfConditioning: true,
|
|
1931
|
+
decoderCacheMode: 'encoder_kv_readonly_canvas_concat',
|
|
1932
|
+
router: {
|
|
1933
|
+
scaleHiddenStates: true,
|
|
1934
|
+
normalizeTopK: true,
|
|
1935
|
+
perExpertScale: true,
|
|
1936
|
+
},
|
|
1937
|
+
};
|
|
1938
|
+
}
|
|
1939
|
+
|
|
1940
|
+
function applyDiffusionGemmaInferenceContract(inference, rawConfig, generationConfig, modelId, resolvedModelType) {
|
|
1941
|
+
if (!resolveDiffusionGemmaConfig(rawConfig, resolvedModelType)) {
|
|
1942
|
+
return inference;
|
|
1943
|
+
}
|
|
1944
|
+
if (inference?.diffusionGemma && typeof inference.diffusionGemma === 'object') {
|
|
1945
|
+
return inference;
|
|
1946
|
+
}
|
|
1947
|
+
return {
|
|
1948
|
+
...inference,
|
|
1949
|
+
diffusionGemma: resolveDiffusionGemmaInferenceContract(rawConfig, generationConfig, modelId),
|
|
1950
|
+
};
|
|
1951
|
+
}
|
|
1952
|
+
|
|
1163
1953
|
|
|
1164
1954
|
export function buildTensorMap(tensors, shardSize) {
|
|
1165
1955
|
if (!shardSize || shardSize <= 0) {
|
|
@@ -1229,14 +2019,6 @@ export function resolveConvertedAt(value) {
|
|
|
1229
2019
|
return new Date(parsed).toISOString();
|
|
1230
2020
|
}
|
|
1231
2021
|
|
|
1232
|
-
function cloneJsonValue(value) {
|
|
1233
|
-
if (value == null) return value;
|
|
1234
|
-
if (typeof structuredClone === 'function') {
|
|
1235
|
-
return structuredClone(value);
|
|
1236
|
-
}
|
|
1237
|
-
return JSON.parse(JSON.stringify(value));
|
|
1238
|
-
}
|
|
1239
|
-
|
|
1240
2022
|
export function resolveManifestMultimodalConfig(rawConfig, manifestConfig = null) {
|
|
1241
2023
|
const explicitVisionConfig = manifestConfig?.visionConfig;
|
|
1242
2024
|
const explicitAudioConfig = manifestConfig?.audioConfig;
|
|
@@ -1248,6 +2030,215 @@ export function resolveManifestMultimodalConfig(rawConfig, manifestConfig = null
|
|
|
1248
2030
|
};
|
|
1249
2031
|
}
|
|
1250
2032
|
|
|
2033
|
+
function isPlainRecord(value) {
|
|
2034
|
+
return value != null && typeof value === 'object' && !Array.isArray(value);
|
|
2035
|
+
}
|
|
2036
|
+
|
|
2037
|
+
function stripUndefined(value) {
|
|
2038
|
+
if (Array.isArray(value)) {
|
|
2039
|
+
return value.map(stripUndefined);
|
|
2040
|
+
}
|
|
2041
|
+
if (!isPlainRecord(value)) {
|
|
2042
|
+
return value;
|
|
2043
|
+
}
|
|
2044
|
+
const out = {};
|
|
2045
|
+
for (const key of Object.keys(value).sort()) {
|
|
2046
|
+
const item = value[key];
|
|
2047
|
+
if (item !== undefined) {
|
|
2048
|
+
out[key] = stripUndefined(item);
|
|
2049
|
+
}
|
|
2050
|
+
}
|
|
2051
|
+
return out;
|
|
2052
|
+
}
|
|
2053
|
+
|
|
2054
|
+
function canonicalJson(value) {
|
|
2055
|
+
return JSON.stringify(stripUndefined(value));
|
|
2056
|
+
}
|
|
2057
|
+
|
|
2058
|
+
function normalizeDigest(value, label) {
|
|
2059
|
+
const digest = typeof value === 'string' ? value.trim() : '';
|
|
2060
|
+
if (!digest) {
|
|
2061
|
+
throw new Error(`Missing ${label} digest`);
|
|
2062
|
+
}
|
|
2063
|
+
return digest.startsWith('sha256:') ? digest : `sha256:${digest}`;
|
|
2064
|
+
}
|
|
2065
|
+
|
|
2066
|
+
function digestSuffix(value, length = 12) {
|
|
2067
|
+
return String(value || '').replace(/^sha256:/, '').slice(0, length);
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2070
|
+
async function hashArtifactValue(hashString, value, label) {
|
|
2071
|
+
if (typeof hashString !== 'function') {
|
|
2072
|
+
return null;
|
|
2073
|
+
}
|
|
2074
|
+
const digest = await hashString(canonicalJson(value));
|
|
2075
|
+
return normalizeDigest(digest, label);
|
|
2076
|
+
}
|
|
2077
|
+
|
|
2078
|
+
function resolveArtifactSourceFormat(options) {
|
|
2079
|
+
const explicit = typeof options?.sourceFormat === 'string' ? options.sourceFormat.trim() : '';
|
|
2080
|
+
if (explicit) return explicit;
|
|
2081
|
+
const sourcePath = typeof options?.sourcePath === 'string' ? options.sourcePath.trim().toLowerCase() : '';
|
|
2082
|
+
if (sourcePath.endsWith('.gguf')) return 'gguf';
|
|
2083
|
+
if (sourcePath.endsWith('.tflite')) return 'tflite';
|
|
2084
|
+
if (sourcePath.endsWith('.task')) return 'task';
|
|
2085
|
+
if (sourcePath.endsWith('.litertlm')) return 'litertlm';
|
|
2086
|
+
return 'safetensors';
|
|
2087
|
+
}
|
|
2088
|
+
|
|
2089
|
+
function resolveSourceCheckpointIdentity(explicit, options) {
|
|
2090
|
+
const sourceRepo = typeof explicit?.sourceRepo === 'string' && explicit.sourceRepo.trim()
|
|
2091
|
+
? explicit.sourceRepo.trim()
|
|
2092
|
+
: null;
|
|
2093
|
+
const sourceRevision = typeof explicit?.sourceRevision === 'string' && explicit.sourceRevision.trim()
|
|
2094
|
+
? explicit.sourceRevision.trim()
|
|
2095
|
+
: null;
|
|
2096
|
+
const sourcePath = typeof options?.sourcePath === 'string' && options.sourcePath.trim()
|
|
2097
|
+
? options.sourcePath.trim()
|
|
2098
|
+
: null;
|
|
2099
|
+
const source = typeof options?.source === 'string' && options.source.trim()
|
|
2100
|
+
? options.source.trim()
|
|
2101
|
+
: null;
|
|
2102
|
+
const sourceCheckpointId = typeof explicit?.sourceCheckpointId === 'string' && explicit.sourceCheckpointId.trim()
|
|
2103
|
+
? explicit.sourceCheckpointId.trim()
|
|
2104
|
+
: (
|
|
2105
|
+
sourceRepo && sourceRevision
|
|
2106
|
+
? `${sourceRepo}@${sourceRevision}`
|
|
2107
|
+
: (sourceRepo ?? source ?? sourcePath ?? null)
|
|
2108
|
+
);
|
|
2109
|
+
return {
|
|
2110
|
+
sourceCheckpointId,
|
|
2111
|
+
sourceRepo,
|
|
2112
|
+
sourceRevision,
|
|
2113
|
+
};
|
|
2114
|
+
}
|
|
2115
|
+
|
|
2116
|
+
function inferArtifactModalitySet(modelType, tensorLocations, converterConfig) {
|
|
2117
|
+
if (modelType === 'diffusion') {
|
|
2118
|
+
return ['image'];
|
|
2119
|
+
}
|
|
2120
|
+
const names = Object.keys(tensorLocations ?? {}).map((name) => name.toLowerCase());
|
|
2121
|
+
const modalities = new Set();
|
|
2122
|
+
if (modelType === 'embedding') {
|
|
2123
|
+
modalities.add('embedding');
|
|
2124
|
+
} else {
|
|
2125
|
+
modalities.add('text');
|
|
2126
|
+
}
|
|
2127
|
+
if (converterConfig?.output?.textOnly === true) {
|
|
2128
|
+
return [...modalities].sort();
|
|
2129
|
+
}
|
|
2130
|
+
if (names.some((name) => name.includes('vision') || name.includes('visual') || name.includes('image'))) {
|
|
2131
|
+
modalities.add('vision');
|
|
2132
|
+
}
|
|
2133
|
+
if (names.some((name) => name.includes('audio'))) {
|
|
2134
|
+
modalities.add('audio');
|
|
2135
|
+
}
|
|
2136
|
+
if (names.some((name) => name.includes('projector') || name.includes('mm_projector'))) {
|
|
2137
|
+
modalities.add('projector');
|
|
2138
|
+
}
|
|
2139
|
+
return [...modalities].sort();
|
|
2140
|
+
}
|
|
2141
|
+
|
|
2142
|
+
function resolveMaterializationProfile(quantizationInfo, inference) {
|
|
2143
|
+
const materialization = typeof inference?.session?.perLayerInputs?.materialization === 'string'
|
|
2144
|
+
? inference.session.perLayerInputs.materialization
|
|
2145
|
+
: 'standard';
|
|
2146
|
+
const perLayerEmbeddings = typeof quantizationInfo?.perLayerEmbeddings === 'string'
|
|
2147
|
+
? quantizationInfo.perLayerEmbeddings
|
|
2148
|
+
: null;
|
|
2149
|
+
return perLayerEmbeddings ? `${materialization}-${perLayerEmbeddings}` : materialization;
|
|
2150
|
+
}
|
|
2151
|
+
|
|
2152
|
+
async function buildArtifactIdentity(options) {
|
|
2153
|
+
const explicit = isPlainRecord(options?.explicitArtifactIdentity)
|
|
2154
|
+
? options.explicitArtifactIdentity
|
|
2155
|
+
: {};
|
|
2156
|
+
const hashString = options?.hashString;
|
|
2157
|
+
if (typeof hashString !== 'function' && Object.keys(explicit).length > 0) {
|
|
2158
|
+
return stripUndefined(explicit);
|
|
2159
|
+
}
|
|
2160
|
+
if (typeof hashString !== 'function') {
|
|
2161
|
+
return null;
|
|
2162
|
+
}
|
|
2163
|
+
|
|
2164
|
+
const sourceIdentity = resolveSourceCheckpointIdentity(explicit, options);
|
|
2165
|
+
const sourceFormat = typeof explicit.sourceFormat === 'string' && explicit.sourceFormat.trim()
|
|
2166
|
+
? explicit.sourceFormat.trim()
|
|
2167
|
+
: resolveArtifactSourceFormat(options);
|
|
2168
|
+
const conversionConfigDigest = explicit.conversionConfigDigest
|
|
2169
|
+
?? (options.conversionConfig
|
|
2170
|
+
? await hashArtifactValue(hashString, options.conversionConfig, 'conversionConfig')
|
|
2171
|
+
: null);
|
|
2172
|
+
const shardSetHash = explicit.shardSetHash ?? explicit.weightPackHash
|
|
2173
|
+
?? await hashArtifactValue(
|
|
2174
|
+
hashString,
|
|
2175
|
+
{
|
|
2176
|
+
hashAlgorithm: options.hashAlgorithm,
|
|
2177
|
+
shards: (options.shards ?? []).map((shard) => ({
|
|
2178
|
+
index: shard.index,
|
|
2179
|
+
filename: shard.filename,
|
|
2180
|
+
size: shard.size,
|
|
2181
|
+
hash: shard.hash,
|
|
2182
|
+
offset: shard.offset,
|
|
2183
|
+
})),
|
|
2184
|
+
},
|
|
2185
|
+
'shardSet'
|
|
2186
|
+
);
|
|
2187
|
+
const modalitySet = Array.isArray(explicit.modalitySet) && explicit.modalitySet.length > 0
|
|
2188
|
+
? [...explicit.modalitySet]
|
|
2189
|
+
: inferArtifactModalitySet(options.modelType, options.tensorLocations, options.converterConfig);
|
|
2190
|
+
const materializationProfile = explicit.materializationProfile
|
|
2191
|
+
?? resolveMaterializationProfile(options.quantizationInfo, options.inference);
|
|
2192
|
+
const weightPackInput = {
|
|
2193
|
+
sourceCheckpointId: sourceIdentity.sourceCheckpointId,
|
|
2194
|
+
sourceFormat,
|
|
2195
|
+
modelType: options.modelType,
|
|
2196
|
+
modalitySet,
|
|
2197
|
+
quantizationInfo: options.quantizationInfo,
|
|
2198
|
+
materializationProfile,
|
|
2199
|
+
shardSetHash,
|
|
2200
|
+
sharding: {
|
|
2201
|
+
shardSizeBytes: options.converterConfig?.sharding?.shardSizeBytes ?? null,
|
|
2202
|
+
},
|
|
2203
|
+
output: {
|
|
2204
|
+
textOnly: options.converterConfig?.output?.textOnly === true,
|
|
2205
|
+
},
|
|
2206
|
+
};
|
|
2207
|
+
const weightPackHash = explicit.weightPackHash
|
|
2208
|
+
?? await hashArtifactValue(hashString, weightPackInput, 'weightPack');
|
|
2209
|
+
const weightPackId = explicit.weightPackId
|
|
2210
|
+
?? `${sanitizeModelId(options.modelId) ?? 'model'}-wp-${digestSuffix(weightPackHash)}`;
|
|
2211
|
+
const manifestVariantHash = await hashArtifactValue(
|
|
2212
|
+
hashString,
|
|
2213
|
+
{
|
|
2214
|
+
weightPackId,
|
|
2215
|
+
modelType: options.modelType,
|
|
2216
|
+
inference: options.inference,
|
|
2217
|
+
config: options.manifestConfig ?? null,
|
|
2218
|
+
},
|
|
2219
|
+
'manifestVariant'
|
|
2220
|
+
);
|
|
2221
|
+
const manifestVariantId = explicit.manifestVariantId
|
|
2222
|
+
?? `${sanitizeModelId(options.modelId) ?? 'model'}-mv-${digestSuffix(manifestVariantHash)}`;
|
|
2223
|
+
|
|
2224
|
+
return stripUndefined({
|
|
2225
|
+
...explicit,
|
|
2226
|
+
sourceCheckpointId: sourceIdentity.sourceCheckpointId,
|
|
2227
|
+
sourceRepo: sourceIdentity.sourceRepo ?? undefined,
|
|
2228
|
+
sourceRevision: sourceIdentity.sourceRevision ?? undefined,
|
|
2229
|
+
sourceFormat,
|
|
2230
|
+
conversionConfigPath: explicit.conversionConfigPath ?? options.conversionConfigPath ?? undefined,
|
|
2231
|
+
conversionConfigDigest: conversionConfigDigest ?? undefined,
|
|
2232
|
+
weightPackId,
|
|
2233
|
+
weightPackHash,
|
|
2234
|
+
shardSetHash,
|
|
2235
|
+
manifestVariantId,
|
|
2236
|
+
modalitySet,
|
|
2237
|
+
materializationProfile,
|
|
2238
|
+
artifactCompleteness: explicit.artifactCompleteness ?? 'complete',
|
|
2239
|
+
});
|
|
2240
|
+
}
|
|
2241
|
+
|
|
1251
2242
|
|
|
1252
2243
|
export function createManifest(
|
|
1253
2244
|
modelId,
|
|
@@ -1289,6 +2280,13 @@ export function createManifest(
|
|
|
1289
2280
|
if (!inference) {
|
|
1290
2281
|
throw new Error('inference config is required — use a v1 conversion config');
|
|
1291
2282
|
}
|
|
2283
|
+
inference = applyDiffusionGemmaInferenceContract(
|
|
2284
|
+
inference,
|
|
2285
|
+
rawConfig,
|
|
2286
|
+
generationConfig,
|
|
2287
|
+
modelId,
|
|
2288
|
+
resolvedModelType
|
|
2289
|
+
);
|
|
1292
2290
|
|
|
1293
2291
|
const embeddingOutput = inferEmbeddingOutputConfig(tensorLocations);
|
|
1294
2292
|
const hasExplicitEmbeddingPostprocessor = Object.prototype.hasOwnProperty.call(
|
|
@@ -1328,7 +2326,8 @@ export function createManifest(
|
|
|
1328
2326
|
throw new Error('Missing hashAlgorithm for manifest');
|
|
1329
2327
|
}
|
|
1330
2328
|
|
|
1331
|
-
const
|
|
2329
|
+
const isTextOnlyArtifact = options.textOnly === true;
|
|
2330
|
+
const multimodalConfig = isDiffusion || isTextOnlyArtifact
|
|
1332
2331
|
? { vision_config: null, audio_config: null }
|
|
1333
2332
|
: resolveManifestMultimodalConfig(rawConfig, manifestPolicy);
|
|
1334
2333
|
const manifestConfig = isDiffusion
|
|
@@ -1344,6 +2343,8 @@ export function createManifest(
|
|
|
1344
2343
|
modelType: resolvedModelType,
|
|
1345
2344
|
quantization: resolvedQuantization,
|
|
1346
2345
|
quantizationInfo: options.quantizationInfo,
|
|
2346
|
+
...(options.artifactIdentity ? { artifactIdentity: options.artifactIdentity } : {}),
|
|
2347
|
+
...(options.weightsRef ? { weightsRef: options.weightsRef } : {}),
|
|
1347
2348
|
architecture: resolvedArchitecture,
|
|
1348
2349
|
moeConfig,
|
|
1349
2350
|
inference,
|
|
@@ -1394,7 +2395,7 @@ export function createManifest(
|
|
|
1394
2395
|
// Main Converter (uses I/O adapter)
|
|
1395
2396
|
// ============================================================================
|
|
1396
2397
|
|
|
1397
|
-
const MAX_TENSOR_TYPED_ARRAY_BYTES =
|
|
2398
|
+
const MAX_TENSOR_TYPED_ARRAY_BYTES = 0x7fff_ffff;
|
|
1398
2399
|
|
|
1399
2400
|
export async function convertModel(model, io, options = {}) {
|
|
1400
2401
|
const { onProgress, signal } = options;
|
|
@@ -1413,7 +2414,17 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1413
2414
|
if (!modelId) {
|
|
1414
2415
|
throw new Error('Missing modelId for conversion');
|
|
1415
2416
|
}
|
|
1416
|
-
const tensors =
|
|
2417
|
+
const tensors = materializeTiedLmHeadTensor(
|
|
2418
|
+
normalizeCompressedTensorsW4A16(
|
|
2419
|
+
resolveConversionTensors(model, converterConfig),
|
|
2420
|
+
converterConfig
|
|
2421
|
+
),
|
|
2422
|
+
{
|
|
2423
|
+
inference: options.inference ?? converterConfig?.inference ?? null,
|
|
2424
|
+
quantizationInfo: options.quantizationInfo ?? null,
|
|
2425
|
+
modelType: options.modelType ?? model.modelType ?? 'transformer',
|
|
2426
|
+
}
|
|
2427
|
+
);
|
|
1417
2428
|
if (!Array.isArray(tensors) || tensors.length === 0) {
|
|
1418
2429
|
const textOnly = converterConfig?.output?.textOnly === true;
|
|
1419
2430
|
if (textOnly) {
|
|
@@ -1536,6 +2547,7 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1536
2547
|
const sourceTensorSize = Number.isFinite(tensor?.size) ? Number(tensor.size) : null;
|
|
1537
2548
|
let outDtype = tensor.dtype;
|
|
1538
2549
|
let outLayout = null;
|
|
2550
|
+
let tensorStorage = null;
|
|
1539
2551
|
let tensorSize = 0;
|
|
1540
2552
|
|
|
1541
2553
|
if (
|
|
@@ -1550,6 +2562,12 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1550
2562
|
}
|
|
1551
2563
|
|
|
1552
2564
|
let emittedChunk = false;
|
|
2565
|
+
// For PLE INT4 per-row quantization, each row-chunk returns its own
|
|
2566
|
+
// per-row F32 scale slice via companionData. Accumulate them across
|
|
2567
|
+
// chunks; after the stream completes, write the concatenated scales
|
|
2568
|
+
// blob and attach sourceTransform.scaleSource pointing at it.
|
|
2569
|
+
const companionChunks = [];
|
|
2570
|
+
let accumulatedSourceTransform = null;
|
|
1553
2571
|
await options.largeTensorTransformer({
|
|
1554
2572
|
tensor,
|
|
1555
2573
|
transformContext,
|
|
@@ -1561,9 +2579,11 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1561
2579
|
}
|
|
1562
2580
|
const chunkOutDtype = result?.outDtype ?? tensor.dtype;
|
|
1563
2581
|
const chunkOutLayout = result?.outLayout ?? null;
|
|
2582
|
+
const chunkStorage = result?.storage ?? null;
|
|
1564
2583
|
if (!emittedChunk) {
|
|
1565
2584
|
outDtype = chunkOutDtype;
|
|
1566
2585
|
outLayout = chunkOutLayout;
|
|
2586
|
+
tensorStorage = chunkStorage;
|
|
1567
2587
|
emittedChunk = true;
|
|
1568
2588
|
} else {
|
|
1569
2589
|
if (chunkOutDtype !== outDtype) {
|
|
@@ -1572,15 +2592,63 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1572
2592
|
if (chunkOutLayout !== outLayout) {
|
|
1573
2593
|
throw new Error(`Large tensor transformer returned inconsistent layout for ${tensor.name}.`);
|
|
1574
2594
|
}
|
|
2595
|
+
if (JSON.stringify(chunkStorage) !== JSON.stringify(tensorStorage)) {
|
|
2596
|
+
throw new Error(`Large tensor transformer returned inconsistent storage descriptor for ${tensor.name}.`);
|
|
2597
|
+
}
|
|
1575
2598
|
}
|
|
1576
2599
|
tensorSize += tensorData.byteLength;
|
|
1577
2600
|
await appendTensorBytes(tensorData, tensorSpans);
|
|
2601
|
+
|
|
2602
|
+
if (result?.companionData instanceof Uint8Array && result.companionData.byteLength > 0) {
|
|
2603
|
+
if (!result.sourceTransform) {
|
|
2604
|
+
throw new Error(
|
|
2605
|
+
`Large tensor chunk returned companionData without sourceTransform for ${tensor.name}.`
|
|
2606
|
+
);
|
|
2607
|
+
}
|
|
2608
|
+
companionChunks.push(result.companionData);
|
|
2609
|
+
if (!accumulatedSourceTransform) {
|
|
2610
|
+
accumulatedSourceTransform = { ...result.sourceTransform };
|
|
2611
|
+
}
|
|
2612
|
+
} else if (result?.sourceTransform && !accumulatedSourceTransform) {
|
|
2613
|
+
accumulatedSourceTransform = { ...result.sourceTransform };
|
|
2614
|
+
}
|
|
1578
2615
|
},
|
|
1579
2616
|
});
|
|
1580
2617
|
|
|
1581
2618
|
if (!emittedChunk) {
|
|
1582
2619
|
throw new Error(`Large tensor transformer did not emit any bytes for ${tensor.name}.`);
|
|
1583
2620
|
}
|
|
2621
|
+
|
|
2622
|
+
if (accumulatedSourceTransform) {
|
|
2623
|
+
let companionBytes = null;
|
|
2624
|
+
if (companionChunks.length > 0) {
|
|
2625
|
+
let totalCompanionBytes = 0;
|
|
2626
|
+
for (const chunk of companionChunks) totalCompanionBytes += chunk.byteLength;
|
|
2627
|
+
companionBytes = new Uint8Array(totalCompanionBytes);
|
|
2628
|
+
let offset = 0;
|
|
2629
|
+
for (const chunk of companionChunks) {
|
|
2630
|
+
companionBytes.set(chunk, offset);
|
|
2631
|
+
offset += chunk.byteLength;
|
|
2632
|
+
}
|
|
2633
|
+
const companionSpans = [];
|
|
2634
|
+
await appendTensorBytes(companionBytes, companionSpans);
|
|
2635
|
+
if (companionSpans.length !== 1) {
|
|
2636
|
+
throw new Error(
|
|
2637
|
+
`Companion scales for ${tensor.name} must land in a single shard (got ${companionSpans.length}).`
|
|
2638
|
+
);
|
|
2639
|
+
}
|
|
2640
|
+
transformContext._pleSourceTransform = {
|
|
2641
|
+
...accumulatedSourceTransform,
|
|
2642
|
+
scaleSource: {
|
|
2643
|
+
shard: companionSpans[0].shardIndex,
|
|
2644
|
+
offset: companionSpans[0].offset,
|
|
2645
|
+
size: companionBytes.byteLength,
|
|
2646
|
+
},
|
|
2647
|
+
};
|
|
2648
|
+
} else {
|
|
2649
|
+
transformContext._pleSourceTransform = accumulatedSourceTransform;
|
|
2650
|
+
}
|
|
2651
|
+
}
|
|
1584
2652
|
} else {
|
|
1585
2653
|
const data = await io.readTensorData(tensor);
|
|
1586
2654
|
const tensorDataInput = new Uint8Array(data);
|
|
@@ -1601,13 +2669,47 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1601
2669
|
}
|
|
1602
2670
|
outDtype = transformResult?.outDtype ?? tensor.dtype;
|
|
1603
2671
|
outLayout = transformResult?.outLayout ?? null;
|
|
2672
|
+
tensorStorage = transformResult?.storage ?? null;
|
|
1604
2673
|
tensorSize = tensorData.byteLength;
|
|
1605
2674
|
await appendTensorBytes(tensorData, tensorSpans);
|
|
2675
|
+
|
|
2676
|
+
// Companion data (e.g., INT4 per-row scales for PLE) writes to its own
|
|
2677
|
+
// spans and is referenced from sourceTransform.scaleSource on the
|
|
2678
|
+
// primary tensor location.
|
|
2679
|
+
const companionData = transformResult?.companionData;
|
|
2680
|
+
if (companionData instanceof Uint8Array && companionData.byteLength > 0) {
|
|
2681
|
+
if (!transformResult?.sourceTransform) {
|
|
2682
|
+
throw new Error(
|
|
2683
|
+
`Tensor transformer returned companionData without sourceTransform for ${tensor.name}.`
|
|
2684
|
+
);
|
|
2685
|
+
}
|
|
2686
|
+
const companionSpans = [];
|
|
2687
|
+
await appendTensorBytes(companionData, companionSpans);
|
|
2688
|
+
if (companionSpans.length !== 1) {
|
|
2689
|
+
throw new Error(
|
|
2690
|
+
`Companion scales for ${tensor.name} must land in a single shard (got ${companionSpans.length}).`
|
|
2691
|
+
);
|
|
2692
|
+
}
|
|
2693
|
+
transformContext._pleSourceTransform = {
|
|
2694
|
+
...transformResult.sourceTransform,
|
|
2695
|
+
scaleSource: {
|
|
2696
|
+
shard: companionSpans[0].shardIndex,
|
|
2697
|
+
offset: companionSpans[0].offset,
|
|
2698
|
+
size: companionData.byteLength,
|
|
2699
|
+
},
|
|
2700
|
+
};
|
|
2701
|
+
} else if (transformResult?.sourceTransform) {
|
|
2702
|
+
transformContext._pleSourceTransform = transformResult.sourceTransform;
|
|
2703
|
+
} else {
|
|
2704
|
+
transformContext._pleSourceTransform = null;
|
|
2705
|
+
}
|
|
1606
2706
|
}
|
|
1607
2707
|
|
|
1608
2708
|
// Record tensor location
|
|
1609
2709
|
const role = resolveTensorRole(tensor);
|
|
1610
2710
|
const group = resolveTensorGroup(tensor, tensorGroupModelType);
|
|
2711
|
+
const pleSourceTransform = transformContext._pleSourceTransform ?? null;
|
|
2712
|
+
transformContext._pleSourceTransform = null;
|
|
1611
2713
|
|
|
1612
2714
|
if (tensorSpans.length === 1) {
|
|
1613
2715
|
tensorLocations[tensor.name] = {
|
|
@@ -1619,6 +2721,8 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1619
2721
|
role,
|
|
1620
2722
|
group,
|
|
1621
2723
|
...(outLayout ? { layout: outLayout } : {}),
|
|
2724
|
+
...(tensorStorage ? { storage: tensorStorage } : {}),
|
|
2725
|
+
...(pleSourceTransform ? { sourceTransform: pleSourceTransform } : {}),
|
|
1622
2726
|
};
|
|
1623
2727
|
} else {
|
|
1624
2728
|
tensorLocations[tensor.name] = {
|
|
@@ -1629,6 +2733,8 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1629
2733
|
role,
|
|
1630
2734
|
group,
|
|
1631
2735
|
...(outLayout ? { layout: outLayout } : {}),
|
|
2736
|
+
...(tensorStorage ? { storage: tensorStorage } : {}),
|
|
2737
|
+
...(pleSourceTransform ? { sourceTransform: pleSourceTransform } : {}),
|
|
1632
2738
|
};
|
|
1633
2739
|
}
|
|
1634
2740
|
|
|
@@ -1662,18 +2768,43 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1662
2768
|
options.quantization ?? model.quantization
|
|
1663
2769
|
);
|
|
1664
2770
|
|
|
2771
|
+
validateInt4PleMaterializationContract(tensorLocations, options.inference, modelId);
|
|
2772
|
+
|
|
2773
|
+
const artifactIdentity = await buildArtifactIdentity({
|
|
2774
|
+
modelId,
|
|
2775
|
+
modelType: options.modelType,
|
|
2776
|
+
source: options.source,
|
|
2777
|
+
sourcePath: options.sourcePath,
|
|
2778
|
+
sourceFormat: options.sourceFormat,
|
|
2779
|
+
conversionConfigPath: options.conversionConfigPath,
|
|
2780
|
+
conversionConfig: options.conversionConfig,
|
|
2781
|
+
explicitArtifactIdentity: converterConfig?.manifest?.artifactIdentity ?? options.artifactIdentity ?? null,
|
|
2782
|
+
hashString: options.hashString,
|
|
2783
|
+
hashAlgorithm: converterConfig.manifest.hashAlgorithm,
|
|
2784
|
+
shards,
|
|
2785
|
+
tensorLocations,
|
|
2786
|
+
quantizationInfo: effectiveQuantizationInfo,
|
|
2787
|
+
inference: options.inference,
|
|
2788
|
+
manifestConfig: converterConfig?.manifest ?? null,
|
|
2789
|
+
converterConfig,
|
|
2790
|
+
});
|
|
2791
|
+
|
|
1665
2792
|
const manifest = createManifest(modelId, model, shards, tensorLocations, {
|
|
1666
|
-
source: 'convert-core',
|
|
2793
|
+
source: options.source ?? 'convert-core',
|
|
1667
2794
|
modelType: options.modelType,
|
|
1668
2795
|
quantization: effectiveManifestQuantization,
|
|
1669
2796
|
quantizationInfo: effectiveQuantizationInfo,
|
|
2797
|
+
moeConfig: converterConfig?.moeConfig ?? options.moeConfig ?? null,
|
|
2798
|
+
artifactIdentity,
|
|
2799
|
+
weightsRef: converterConfig?.manifest?.weightsRef ?? options.weightsRef ?? null,
|
|
1670
2800
|
hashAlgorithm: converterConfig.manifest.hashAlgorithm,
|
|
1671
2801
|
architecture: options.architecture,
|
|
1672
2802
|
inference: options.inference,
|
|
1673
|
-
eosTokenId: options.eosTokenId,
|
|
2803
|
+
eosTokenId: converterConfig?.manifest?.eosTokenId ?? options.eosTokenId,
|
|
1674
2804
|
convertedAt: converterConfig?.manifest?.conversion?.convertedAt ?? null,
|
|
1675
2805
|
conversionInfo: converterConfig?.manifest?.conversion ?? null,
|
|
1676
2806
|
manifestConfig: converterConfig?.manifest ?? null,
|
|
2807
|
+
textOnly: converterConfig?.output?.textOnly === true,
|
|
1677
2808
|
});
|
|
1678
2809
|
|
|
1679
2810
|
// Write manifest
|