doppler-gpu 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +19 -15
- package/package.json +77 -29
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +64 -6
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +524 -350
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +12 -2
- package/src/config/schema/doppler.schema.js +108 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +68 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +76 -0
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +107 -0
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +3 -21
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +6 -5
- package/src/gpu/kernels/index.js +3 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +12 -0
- package/src/inference/browser-harness-text-helpers.js +223 -14
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +264 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +22 -27
- package/src/inference/pipelines/text/embed.js +0 -39
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +4 -5
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +8 -3
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +223 -547
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +5 -2
- package/src/inference/pipelines/text/init.js +183 -724
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.js +2 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +120 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +223 -27
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +13 -3
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +11 -3
- package/src/storage/artifact-storage-context.js +231 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +21 -24
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +2 -0
- package/src/tooling/source-runtime-bundle.js +3 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
package/src/converter/core.js
CHANGED
|
@@ -38,7 +38,9 @@ import {
|
|
|
38
38
|
quantizeToQ4KM,
|
|
39
39
|
quantizeToQ4KMRowWise,
|
|
40
40
|
quantizeToQ4KMColumnWise,
|
|
41
|
+
quantizeToInt4PerRowSymmetric,
|
|
41
42
|
} from './quantizer.js';
|
|
43
|
+
import { cloneJsonValue } from '../utils/clone-json.js';
|
|
42
44
|
|
|
43
45
|
// ============================================================================
|
|
44
46
|
// Re-exports for Backward Compatibility
|
|
@@ -255,7 +257,16 @@ function resolveConversionTensors(model, converterConfig) {
|
|
|
255
257
|
if (hasLanguageModelNamespace) {
|
|
256
258
|
return source.filter((tensor) => {
|
|
257
259
|
const lower = normalizeTensorName(tensor).toLowerCase();
|
|
258
|
-
|
|
260
|
+
// Keep top-level lm_head/output tensors alongside the language_model.*
|
|
261
|
+
// namespace. Multimodal HF models (e.g. Qwen 3.6-27B) place the
|
|
262
|
+
// language model body under model.language_model.* but expose the
|
|
263
|
+
// language modeling head at the bare top level (`lm_head.weight`).
|
|
264
|
+
// Dropping it leaves text-only conversion without an LM head and
|
|
265
|
+
// pipeline init fails at loadWeights.
|
|
266
|
+
return lower.startsWith('language_model.')
|
|
267
|
+
|| lower.startsWith('model.language_model.')
|
|
268
|
+
|| lower === 'lm_head.weight'
|
|
269
|
+
|| lower === 'model.lm_head.weight';
|
|
259
270
|
});
|
|
260
271
|
}
|
|
261
272
|
|
|
@@ -264,6 +275,77 @@ function resolveConversionTensors(model, converterConfig) {
|
|
|
264
275
|
));
|
|
265
276
|
}
|
|
266
277
|
|
|
278
|
+
function shouldMaterializeTiedLmHead(tensors, options) {
|
|
279
|
+
if (options?.inference?.output?.tieWordEmbeddings !== true) {
|
|
280
|
+
return false;
|
|
281
|
+
}
|
|
282
|
+
if (normalizeStorageQuant(options?.quantizationInfo?.lmHead ?? null) !== 'q4k') {
|
|
283
|
+
return false;
|
|
284
|
+
}
|
|
285
|
+
const embeddingQuant = normalizeStorageQuant(options?.quantizationInfo?.embeddings ?? null);
|
|
286
|
+
if (embeddingQuant === 'q4k') {
|
|
287
|
+
return false;
|
|
288
|
+
}
|
|
289
|
+
return !tensors.some((tensor) => resolveTensorRole(tensor) === 'lm_head');
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
function resolveTiedLmHeadName(embeddingName) {
|
|
293
|
+
const name = typeof embeddingName === 'string' ? embeddingName.trim() : '';
|
|
294
|
+
if (name === 'model.language_model.model.embed_tokens.weight') {
|
|
295
|
+
return 'model.language_model.lm_head.weight';
|
|
296
|
+
}
|
|
297
|
+
if (name === 'language_model.model.embed_tokens.weight') {
|
|
298
|
+
return 'language_model.lm_head.weight';
|
|
299
|
+
}
|
|
300
|
+
if (name === 'model.language_model.embed_tokens.weight') {
|
|
301
|
+
return 'model.language_model.lm_head.weight';
|
|
302
|
+
}
|
|
303
|
+
if (name === 'language_model.embed_tokens.weight') {
|
|
304
|
+
return 'language_model.lm_head.weight';
|
|
305
|
+
}
|
|
306
|
+
if (name === 'model.embed_tokens.weight' || name === 'embed_tokens.weight') {
|
|
307
|
+
return 'lm_head.weight';
|
|
308
|
+
}
|
|
309
|
+
return 'lm_head.weight';
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
function resolveTiedEmbeddingTensor(tensors, modelType) {
|
|
313
|
+
const candidates = tensors.filter((tensor) => (
|
|
314
|
+
resolveTensorRole(tensor) === 'embedding'
|
|
315
|
+
&& resolveTensorGroup(tensor, modelType) === 'embed'
|
|
316
|
+
&& Array.isArray(tensor?.shape)
|
|
317
|
+
&& tensor.shape.length === 2
|
|
318
|
+
));
|
|
319
|
+
return candidates[0] ?? null;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
function materializeTiedLmHeadTensor(tensors, options) {
|
|
323
|
+
if (!shouldMaterializeTiedLmHead(tensors, options)) {
|
|
324
|
+
return tensors;
|
|
325
|
+
}
|
|
326
|
+
const embedding = resolveTiedEmbeddingTensor(tensors, options?.modelType ?? 'transformer');
|
|
327
|
+
if (!embedding) {
|
|
328
|
+
throw new Error(
|
|
329
|
+
'Cannot materialize tied Q4K LM head: no 2D token embedding tensor was selected. '
|
|
330
|
+
+ 'Check inference.output.tieWordEmbeddings and the conversion tensor filter.'
|
|
331
|
+
);
|
|
332
|
+
}
|
|
333
|
+
const name = resolveTiedLmHeadName(embedding.name);
|
|
334
|
+
if (tensors.some((tensor) => tensor?.name === name)) {
|
|
335
|
+
throw new Error(`Cannot materialize tied Q4K LM head: synthetic tensor name "${name}" already exists.`);
|
|
336
|
+
}
|
|
337
|
+
return [
|
|
338
|
+
...tensors,
|
|
339
|
+
{
|
|
340
|
+
...embedding,
|
|
341
|
+
name,
|
|
342
|
+
role: 'lm_head',
|
|
343
|
+
group: 'head',
|
|
344
|
+
sourceTensorName: embedding.name,
|
|
345
|
+
},
|
|
346
|
+
];
|
|
347
|
+
}
|
|
348
|
+
|
|
267
349
|
function toFloat32ForQ4K(tensorData, sourceDtype, tensorName) {
|
|
268
350
|
const dtype = String(sourceDtype || '').toUpperCase();
|
|
269
351
|
if (dtype === 'F32') {
|
|
@@ -722,6 +804,143 @@ export function shouldQuantize(tensorName, shape, options = {}) {
|
|
|
722
804
|
return true;
|
|
723
805
|
}
|
|
724
806
|
|
|
807
|
+
const GEMMA4_PLE_TENSOR_SUFFIXES = [
|
|
808
|
+
'embed_tokens_per_layer.weight',
|
|
809
|
+
'per_layer_embeddings.weight',
|
|
810
|
+
// GGUF (unsloth / ggml-org) naming for Gemma 4's PLE table.
|
|
811
|
+
'per_layer_token_embd.weight',
|
|
812
|
+
];
|
|
813
|
+
|
|
814
|
+
export function isGemma4PerLayerEmbedTensor(tensorName) {
|
|
815
|
+
if (typeof tensorName !== 'string') return false;
|
|
816
|
+
const normalized = tensorName.trim().toLowerCase();
|
|
817
|
+
return GEMMA4_PLE_TENSOR_SUFFIXES.some((suffix) => normalized.endsWith(suffix));
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
function validateInt4PleMaterializationContract(tensorLocations, inference, modelId) {
|
|
821
|
+
const materialization = inference?.session?.perLayerInputs?.materialization;
|
|
822
|
+
if (materialization !== 'gpu_split_tables') {
|
|
823
|
+
return;
|
|
824
|
+
}
|
|
825
|
+
for (const [name, location] of Object.entries(tensorLocations ?? {})) {
|
|
826
|
+
const sourceTransform = location?.sourceTransform ?? null;
|
|
827
|
+
if (
|
|
828
|
+
isGemma4PerLayerEmbedTensor(name)
|
|
829
|
+
&& sourceTransform?.kind === 'litert_axis_dequant'
|
|
830
|
+
&& String(sourceTransform?.sourceDtype ?? '').toUpperCase() === 'INT4'
|
|
831
|
+
) {
|
|
832
|
+
throw new Error(
|
|
833
|
+
`Manifest "${modelId}" Gemma 4 INT4 PLE tensor "${name}" cannot use ` +
|
|
834
|
+
'inference.session.perLayerInputs.materialization="gpu_split_tables". ' +
|
|
835
|
+
'Use materialization="range_backed" or disable INT4 PLE quantization.'
|
|
836
|
+
);
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
function resolveOriginalTensorShape(options) {
|
|
842
|
+
const shape = options?.originalTensorShape;
|
|
843
|
+
if (!Array.isArray(shape) || shape.length !== 2) {
|
|
844
|
+
return null;
|
|
845
|
+
}
|
|
846
|
+
const rows = Number(shape[0]);
|
|
847
|
+
const cols = Number(shape[1]);
|
|
848
|
+
if (!Number.isInteger(rows) || rows <= 0 || !Number.isInteger(cols) || cols <= 0) {
|
|
849
|
+
return null;
|
|
850
|
+
}
|
|
851
|
+
return [rows, cols];
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
function resolvePerLayerEmbeddingQuant(options) {
|
|
855
|
+
const value = (
|
|
856
|
+
options?.perLayerEmbeddings
|
|
857
|
+
?? options?.quantizationInfo?.perLayerEmbeddings
|
|
858
|
+
?? null
|
|
859
|
+
);
|
|
860
|
+
if (value == null) return null;
|
|
861
|
+
return String(value).trim().toLowerCase().replace(/\s+/g, '_') || null;
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
function canInt4QuantizePerRow(tensor, options) {
|
|
865
|
+
if (options?.skipInt4PlePerRow === true) return false;
|
|
866
|
+
if (resolvePerLayerEmbeddingQuant(options) !== 'int4_per_row') return false;
|
|
867
|
+
if (!Array.isArray(tensor.shape) || tensor.shape.length !== 2) return false;
|
|
868
|
+
const [rows, cols] = tensor.shape;
|
|
869
|
+
if (!Number.isInteger(rows) || rows <= 0) return false;
|
|
870
|
+
if (!Number.isInteger(cols) || cols <= 0) return false;
|
|
871
|
+
if ((cols & 1) !== 0) return false;
|
|
872
|
+
const originalShape = resolveOriginalTensorShape(options);
|
|
873
|
+
if (originalShape && originalShape[1] !== cols) return false;
|
|
874
|
+
// Per-row symmetric INT4 produces one scale per row. PLE semantics require
|
|
875
|
+
// one scale per VOCAB token (typically 262144 for Gemma 4). HF stores PLE
|
|
876
|
+
// as [vocab, hidden] (rows = vocab ≫ cols). GGUF stores it transposed as
|
|
877
|
+
// [hidden, vocab] (rows = hidden ≪ cols). Quantizing the GGUF layout with
|
|
878
|
+
// per-row would wrongly share a single scale across all vocab tokens.
|
|
879
|
+
// Require the tensor to already be in [vocab, hidden] layout. Callers that
|
|
880
|
+
// have a GGUF PLE must transpose before reaching this function.
|
|
881
|
+
if (rows <= cols && (!originalShape || originalShape[0] <= originalShape[1])) return false;
|
|
882
|
+
const srcDtype = String(tensor.dtype || '').toUpperCase();
|
|
883
|
+
return srcDtype === 'F32' || srcDtype === 'F16' || srcDtype === 'BF16';
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
function toFloat32FromTensor(bytes, sourceDtype, tensorName) {
|
|
887
|
+
const src = String(sourceDtype || '').toUpperCase();
|
|
888
|
+
if (src === 'F32') {
|
|
889
|
+
if (bytes.byteLength % 4 !== 0) {
|
|
890
|
+
throw new Error(`Invalid F32 tensor byte length for ${tensorName}: ${bytes.byteLength}`);
|
|
891
|
+
}
|
|
892
|
+
return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
|
|
893
|
+
}
|
|
894
|
+
if (src === 'F16') {
|
|
895
|
+
if (bytes.byteLength % 2 !== 0) {
|
|
896
|
+
throw new Error(`Invalid F16 tensor byte length for ${tensorName}: ${bytes.byteLength}`);
|
|
897
|
+
}
|
|
898
|
+
const src16 = new Uint16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2);
|
|
899
|
+
const out = new Float32Array(src16.length);
|
|
900
|
+
for (let i = 0; i < src16.length; i++) out[i] = float16ToFloat32(src16[i]);
|
|
901
|
+
return out;
|
|
902
|
+
}
|
|
903
|
+
if (src === 'BF16') {
|
|
904
|
+
if (bytes.byteLength % 2 !== 0) {
|
|
905
|
+
throw new Error(`Invalid BF16 tensor byte length for ${tensorName}: ${bytes.byteLength}`);
|
|
906
|
+
}
|
|
907
|
+
const src16 = new Uint16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2);
|
|
908
|
+
const out = new Float32Array(src16.length);
|
|
909
|
+
for (let i = 0; i < src16.length; i++) out[i] = bf16ToFloat32(src16[i]);
|
|
910
|
+
return out;
|
|
911
|
+
}
|
|
912
|
+
throw new Error(`Unsupported source dtype "${sourceDtype}" for PLE INT4 quantization of ${tensorName}`);
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
function buildInt4PerRowPleTransform(tensor, bytes, sourceDtype, options = {}) {
|
|
916
|
+
const f32 = toFloat32FromTensor(bytes, sourceDtype, tensor.name);
|
|
917
|
+
const { quantized, scales } = quantizeToInt4PerRowSymmetric(f32, tensor.shape);
|
|
918
|
+
const scalesBytes = new Uint8Array(scales.buffer, scales.byteOffset, scales.byteLength);
|
|
919
|
+
const [rows, cols] = resolveOriginalTensorShape(options) ?? tensor.shape;
|
|
920
|
+
return {
|
|
921
|
+
tensorData: quantized,
|
|
922
|
+
companionData: scalesBytes,
|
|
923
|
+
// outDtype is the LOGICAL dtype the tensor resolves to after dequant at
|
|
924
|
+
// load time. Storage dtype (INT4) is carried in sourceTransform.sourceDtype.
|
|
925
|
+
outDtype: 'F16',
|
|
926
|
+
outLayout: 'row',
|
|
927
|
+
sourceDtype: String(sourceDtype || '').toUpperCase(),
|
|
928
|
+
tensorTargetQuant: 'int4_per_row_ple',
|
|
929
|
+
sourceTransform: {
|
|
930
|
+
kind: 'litert_axis_dequant',
|
|
931
|
+
scheme: 'per_axis_affine',
|
|
932
|
+
sourceDtype: 'INT4',
|
|
933
|
+
targetDtype: 'F16',
|
|
934
|
+
storageEncoding: 'offset_binary',
|
|
935
|
+
scaleSemantics: 'step',
|
|
936
|
+
storageShape: [rows, cols],
|
|
937
|
+
quantAxis: 1,
|
|
938
|
+
// scaleSource gets filled in by the writer with the scales companion's
|
|
939
|
+
// shard/offset/size after appendTensorBytes().
|
|
940
|
+
},
|
|
941
|
+
};
|
|
942
|
+
}
|
|
943
|
+
|
|
725
944
|
export function transformTensorBytes(tensor, rawData, options = {}) {
|
|
726
945
|
const tensorDataInput = rawData instanceof Uint8Array ? rawData : new Uint8Array(rawData);
|
|
727
946
|
let tensorData = tensorDataInput;
|
|
@@ -750,6 +969,15 @@ export function transformTensorBytes(tensor, rawData, options = {}) {
|
|
|
750
969
|
: null
|
|
751
970
|
);
|
|
752
971
|
|
|
972
|
+
// Gemma 4 per-layer embeddings use MediaPipe's INT4 per-row symmetric
|
|
973
|
+
// quantization (verified against gemma-4-E2B-it.litertlm composites:
|
|
974
|
+
// quantizedDimension=0, one F32 scale per vocab row, zero_point=0). Saves
|
|
975
|
+
// ~3.5 GB per model vs the default F16 path. Runtime reads via
|
|
976
|
+
// sourceTransform.kind=litert_axis_dequant with scaleSemantics=step.
|
|
977
|
+
if (isGemma4PerLayerEmbedTensor(tensor.name) && canInt4QuantizePerRow(tensor, options)) {
|
|
978
|
+
return buildInt4PerRowPleTransform(tensor, tensorDataInput, sourceDtype, options);
|
|
979
|
+
}
|
|
980
|
+
|
|
753
981
|
if (tensorTargetQuant === 'q4k') {
|
|
754
982
|
const sourceQuant = normalizeStorageQuant(sourceDtype);
|
|
755
983
|
if (sourceQuant === 'q4k') {
|
|
@@ -963,6 +1191,7 @@ export function extractArchitecture(config, ggufConfig) {
|
|
|
963
1191
|
'num_attention_heads'
|
|
964
1192
|
);
|
|
965
1193
|
const numKVHeads = fromConfig('num_key_value_heads', 'num_kv_heads') ?? numHeads;
|
|
1194
|
+
const numGlobalKVHeads = fromConfig('num_global_key_value_heads', 'num_global_kv_heads');
|
|
966
1195
|
const headDimFromConfig = fromConfig('head_dim') ?? Math.floor(hiddenSize / numHeads);
|
|
967
1196
|
const vocabSize = requireNumber(
|
|
968
1197
|
fromConfig('vocab_size', 'n_vocab'),
|
|
@@ -1006,6 +1235,7 @@ export function extractArchitecture(config, ggufConfig) {
|
|
|
1006
1235
|
intermediateSize,
|
|
1007
1236
|
numAttentionHeads: numHeads,
|
|
1008
1237
|
numKeyValueHeads: numKVHeads,
|
|
1238
|
+
numGlobalKeyValueHeads: numGlobalKVHeads ?? undefined,
|
|
1009
1239
|
headDim: headDimFromConfig,
|
|
1010
1240
|
vocabSize,
|
|
1011
1241
|
maxSeqLen,
|
|
@@ -1052,15 +1282,35 @@ export function extractArchitecture(config, ggufConfig) {
|
|
|
1052
1282
|
'contextLength'
|
|
1053
1283
|
);
|
|
1054
1284
|
|
|
1285
|
+
// Gemma 4-specific fields (optional — undefined on non-Gemma-4 GGUFs).
|
|
1286
|
+
// key_length is per-head; pick the larger of (key_length, key_length_swa)
|
|
1287
|
+
// for globalHeadDim and the smaller for headDim. Matches the mixed-geometry
|
|
1288
|
+
// KV cache expected by src/inference/pipelines/text/layer.js.
|
|
1289
|
+
const keyLen = firstNumber(c.attentionKeyLength);
|
|
1290
|
+
const keyLenSwa = firstNumber(c.attentionKeyLengthSwa);
|
|
1291
|
+
const headDimFromGguf = keyLenSwa != null && keyLen != null
|
|
1292
|
+
? Math.min(keyLen, keyLenSwa)
|
|
1293
|
+
: (keyLenSwa ?? Math.floor(hiddenSize / numHeads));
|
|
1294
|
+
const globalHeadDim = (keyLen != null && keyLenSwa != null && keyLen !== keyLenSwa)
|
|
1295
|
+
? Math.max(keyLen, keyLenSwa)
|
|
1296
|
+
: undefined;
|
|
1297
|
+
const numKvSharedLayers = firstNumber(c.numKvSharedLayers);
|
|
1298
|
+
const hiddenSizePerLayerInput = firstNumber(c.hiddenSizePerLayerInput);
|
|
1299
|
+
const vocabSizePerLayerInput = hiddenSizePerLayerInput != null ? vocabSize : undefined;
|
|
1300
|
+
|
|
1055
1301
|
return {
|
|
1056
1302
|
numLayers,
|
|
1057
1303
|
hiddenSize,
|
|
1058
1304
|
intermediateSize,
|
|
1059
1305
|
numAttentionHeads: numHeads,
|
|
1060
1306
|
numKeyValueHeads: numKVHeads,
|
|
1061
|
-
headDim:
|
|
1307
|
+
headDim: headDimFromGguf,
|
|
1062
1308
|
vocabSize,
|
|
1063
1309
|
maxSeqLen,
|
|
1310
|
+
...(globalHeadDim != null ? { globalHeadDim } : {}),
|
|
1311
|
+
...(numKvSharedLayers != null ? { numKvSharedLayers } : {}),
|
|
1312
|
+
...(hiddenSizePerLayerInput != null ? { hiddenSizePerLayerInput } : {}),
|
|
1313
|
+
...(vocabSizePerLayerInput != null ? { vocabSizePerLayerInput } : {}),
|
|
1064
1314
|
};
|
|
1065
1315
|
}
|
|
1066
1316
|
|
|
@@ -1229,14 +1479,6 @@ export function resolveConvertedAt(value) {
|
|
|
1229
1479
|
return new Date(parsed).toISOString();
|
|
1230
1480
|
}
|
|
1231
1481
|
|
|
1232
|
-
function cloneJsonValue(value) {
|
|
1233
|
-
if (value == null) return value;
|
|
1234
|
-
if (typeof structuredClone === 'function') {
|
|
1235
|
-
return structuredClone(value);
|
|
1236
|
-
}
|
|
1237
|
-
return JSON.parse(JSON.stringify(value));
|
|
1238
|
-
}
|
|
1239
|
-
|
|
1240
1482
|
export function resolveManifestMultimodalConfig(rawConfig, manifestConfig = null) {
|
|
1241
1483
|
const explicitVisionConfig = manifestConfig?.visionConfig;
|
|
1242
1484
|
const explicitAudioConfig = manifestConfig?.audioConfig;
|
|
@@ -1248,6 +1490,215 @@ export function resolveManifestMultimodalConfig(rawConfig, manifestConfig = null
|
|
|
1248
1490
|
};
|
|
1249
1491
|
}
|
|
1250
1492
|
|
|
1493
|
+
function isPlainRecord(value) {
|
|
1494
|
+
return value != null && typeof value === 'object' && !Array.isArray(value);
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
function stripUndefined(value) {
|
|
1498
|
+
if (Array.isArray(value)) {
|
|
1499
|
+
return value.map(stripUndefined);
|
|
1500
|
+
}
|
|
1501
|
+
if (!isPlainRecord(value)) {
|
|
1502
|
+
return value;
|
|
1503
|
+
}
|
|
1504
|
+
const out = {};
|
|
1505
|
+
for (const key of Object.keys(value).sort()) {
|
|
1506
|
+
const item = value[key];
|
|
1507
|
+
if (item !== undefined) {
|
|
1508
|
+
out[key] = stripUndefined(item);
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1511
|
+
return out;
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
function canonicalJson(value) {
|
|
1515
|
+
return JSON.stringify(stripUndefined(value));
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
function normalizeDigest(value, label) {
|
|
1519
|
+
const digest = typeof value === 'string' ? value.trim() : '';
|
|
1520
|
+
if (!digest) {
|
|
1521
|
+
throw new Error(`Missing ${label} digest`);
|
|
1522
|
+
}
|
|
1523
|
+
return digest.startsWith('sha256:') ? digest : `sha256:${digest}`;
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
function digestSuffix(value, length = 12) {
|
|
1527
|
+
return String(value || '').replace(/^sha256:/, '').slice(0, length);
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
async function hashArtifactValue(hashString, value, label) {
|
|
1531
|
+
if (typeof hashString !== 'function') {
|
|
1532
|
+
return null;
|
|
1533
|
+
}
|
|
1534
|
+
const digest = await hashString(canonicalJson(value));
|
|
1535
|
+
return normalizeDigest(digest, label);
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
function resolveArtifactSourceFormat(options) {
|
|
1539
|
+
const explicit = typeof options?.sourceFormat === 'string' ? options.sourceFormat.trim() : '';
|
|
1540
|
+
if (explicit) return explicit;
|
|
1541
|
+
const sourcePath = typeof options?.sourcePath === 'string' ? options.sourcePath.trim().toLowerCase() : '';
|
|
1542
|
+
if (sourcePath.endsWith('.gguf')) return 'gguf';
|
|
1543
|
+
if (sourcePath.endsWith('.tflite')) return 'tflite';
|
|
1544
|
+
if (sourcePath.endsWith('.task')) return 'task';
|
|
1545
|
+
if (sourcePath.endsWith('.litertlm')) return 'litertlm';
|
|
1546
|
+
return 'safetensors';
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1549
|
+
function resolveSourceCheckpointIdentity(explicit, options) {
|
|
1550
|
+
const sourceRepo = typeof explicit?.sourceRepo === 'string' && explicit.sourceRepo.trim()
|
|
1551
|
+
? explicit.sourceRepo.trim()
|
|
1552
|
+
: null;
|
|
1553
|
+
const sourceRevision = typeof explicit?.sourceRevision === 'string' && explicit.sourceRevision.trim()
|
|
1554
|
+
? explicit.sourceRevision.trim()
|
|
1555
|
+
: null;
|
|
1556
|
+
const sourcePath = typeof options?.sourcePath === 'string' && options.sourcePath.trim()
|
|
1557
|
+
? options.sourcePath.trim()
|
|
1558
|
+
: null;
|
|
1559
|
+
const source = typeof options?.source === 'string' && options.source.trim()
|
|
1560
|
+
? options.source.trim()
|
|
1561
|
+
: null;
|
|
1562
|
+
const sourceCheckpointId = typeof explicit?.sourceCheckpointId === 'string' && explicit.sourceCheckpointId.trim()
|
|
1563
|
+
? explicit.sourceCheckpointId.trim()
|
|
1564
|
+
: (
|
|
1565
|
+
sourceRepo && sourceRevision
|
|
1566
|
+
? `${sourceRepo}@${sourceRevision}`
|
|
1567
|
+
: (sourceRepo ?? source ?? sourcePath ?? null)
|
|
1568
|
+
);
|
|
1569
|
+
return {
|
|
1570
|
+
sourceCheckpointId,
|
|
1571
|
+
sourceRepo,
|
|
1572
|
+
sourceRevision,
|
|
1573
|
+
};
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
function inferArtifactModalitySet(modelType, tensorLocations, converterConfig) {
|
|
1577
|
+
if (modelType === 'diffusion') {
|
|
1578
|
+
return ['image'];
|
|
1579
|
+
}
|
|
1580
|
+
const names = Object.keys(tensorLocations ?? {}).map((name) => name.toLowerCase());
|
|
1581
|
+
const modalities = new Set();
|
|
1582
|
+
if (modelType === 'embedding') {
|
|
1583
|
+
modalities.add('embedding');
|
|
1584
|
+
} else {
|
|
1585
|
+
modalities.add('text');
|
|
1586
|
+
}
|
|
1587
|
+
if (converterConfig?.output?.textOnly === true) {
|
|
1588
|
+
return [...modalities].sort();
|
|
1589
|
+
}
|
|
1590
|
+
if (names.some((name) => name.includes('vision') || name.includes('visual') || name.includes('image'))) {
|
|
1591
|
+
modalities.add('vision');
|
|
1592
|
+
}
|
|
1593
|
+
if (names.some((name) => name.includes('audio'))) {
|
|
1594
|
+
modalities.add('audio');
|
|
1595
|
+
}
|
|
1596
|
+
if (names.some((name) => name.includes('projector') || name.includes('mm_projector'))) {
|
|
1597
|
+
modalities.add('projector');
|
|
1598
|
+
}
|
|
1599
|
+
return [...modalities].sort();
|
|
1600
|
+
}
|
|
1601
|
+
|
|
1602
|
+
function resolveMaterializationProfile(quantizationInfo, inference) {
|
|
1603
|
+
const materialization = typeof inference?.session?.perLayerInputs?.materialization === 'string'
|
|
1604
|
+
? inference.session.perLayerInputs.materialization
|
|
1605
|
+
: 'standard';
|
|
1606
|
+
const perLayerEmbeddings = typeof quantizationInfo?.perLayerEmbeddings === 'string'
|
|
1607
|
+
? quantizationInfo.perLayerEmbeddings
|
|
1608
|
+
: null;
|
|
1609
|
+
return perLayerEmbeddings ? `${materialization}-${perLayerEmbeddings}` : materialization;
|
|
1610
|
+
}
|
|
1611
|
+
|
|
1612
|
+
async function buildArtifactIdentity(options) {
|
|
1613
|
+
const explicit = isPlainRecord(options?.explicitArtifactIdentity)
|
|
1614
|
+
? options.explicitArtifactIdentity
|
|
1615
|
+
: {};
|
|
1616
|
+
const hashString = options?.hashString;
|
|
1617
|
+
if (typeof hashString !== 'function' && Object.keys(explicit).length > 0) {
|
|
1618
|
+
return stripUndefined(explicit);
|
|
1619
|
+
}
|
|
1620
|
+
if (typeof hashString !== 'function') {
|
|
1621
|
+
return null;
|
|
1622
|
+
}
|
|
1623
|
+
|
|
1624
|
+
const sourceIdentity = resolveSourceCheckpointIdentity(explicit, options);
|
|
1625
|
+
const sourceFormat = typeof explicit.sourceFormat === 'string' && explicit.sourceFormat.trim()
|
|
1626
|
+
? explicit.sourceFormat.trim()
|
|
1627
|
+
: resolveArtifactSourceFormat(options);
|
|
1628
|
+
const conversionConfigDigest = explicit.conversionConfigDigest
|
|
1629
|
+
?? (options.conversionConfig
|
|
1630
|
+
? await hashArtifactValue(hashString, options.conversionConfig, 'conversionConfig')
|
|
1631
|
+
: null);
|
|
1632
|
+
const shardSetHash = explicit.shardSetHash ?? explicit.weightPackHash
|
|
1633
|
+
?? await hashArtifactValue(
|
|
1634
|
+
hashString,
|
|
1635
|
+
{
|
|
1636
|
+
hashAlgorithm: options.hashAlgorithm,
|
|
1637
|
+
shards: (options.shards ?? []).map((shard) => ({
|
|
1638
|
+
index: shard.index,
|
|
1639
|
+
filename: shard.filename,
|
|
1640
|
+
size: shard.size,
|
|
1641
|
+
hash: shard.hash,
|
|
1642
|
+
offset: shard.offset,
|
|
1643
|
+
})),
|
|
1644
|
+
},
|
|
1645
|
+
'shardSet'
|
|
1646
|
+
);
|
|
1647
|
+
const modalitySet = Array.isArray(explicit.modalitySet) && explicit.modalitySet.length > 0
|
|
1648
|
+
? [...explicit.modalitySet]
|
|
1649
|
+
: inferArtifactModalitySet(options.modelType, options.tensorLocations, options.converterConfig);
|
|
1650
|
+
const materializationProfile = explicit.materializationProfile
|
|
1651
|
+
?? resolveMaterializationProfile(options.quantizationInfo, options.inference);
|
|
1652
|
+
const weightPackInput = {
|
|
1653
|
+
sourceCheckpointId: sourceIdentity.sourceCheckpointId,
|
|
1654
|
+
sourceFormat,
|
|
1655
|
+
modelType: options.modelType,
|
|
1656
|
+
modalitySet,
|
|
1657
|
+
quantizationInfo: options.quantizationInfo,
|
|
1658
|
+
materializationProfile,
|
|
1659
|
+
shardSetHash,
|
|
1660
|
+
sharding: {
|
|
1661
|
+
shardSizeBytes: options.converterConfig?.sharding?.shardSizeBytes ?? null,
|
|
1662
|
+
},
|
|
1663
|
+
output: {
|
|
1664
|
+
textOnly: options.converterConfig?.output?.textOnly === true,
|
|
1665
|
+
},
|
|
1666
|
+
};
|
|
1667
|
+
const weightPackHash = explicit.weightPackHash
|
|
1668
|
+
?? await hashArtifactValue(hashString, weightPackInput, 'weightPack');
|
|
1669
|
+
const weightPackId = explicit.weightPackId
|
|
1670
|
+
?? `${sanitizeModelId(options.modelId) ?? 'model'}-wp-${digestSuffix(weightPackHash)}`;
|
|
1671
|
+
const manifestVariantHash = await hashArtifactValue(
|
|
1672
|
+
hashString,
|
|
1673
|
+
{
|
|
1674
|
+
weightPackId,
|
|
1675
|
+
modelType: options.modelType,
|
|
1676
|
+
inference: options.inference,
|
|
1677
|
+
config: options.manifestConfig ?? null,
|
|
1678
|
+
},
|
|
1679
|
+
'manifestVariant'
|
|
1680
|
+
);
|
|
1681
|
+
const manifestVariantId = explicit.manifestVariantId
|
|
1682
|
+
?? `${sanitizeModelId(options.modelId) ?? 'model'}-mv-${digestSuffix(manifestVariantHash)}`;
|
|
1683
|
+
|
|
1684
|
+
return stripUndefined({
|
|
1685
|
+
...explicit,
|
|
1686
|
+
sourceCheckpointId: sourceIdentity.sourceCheckpointId,
|
|
1687
|
+
sourceRepo: sourceIdentity.sourceRepo ?? undefined,
|
|
1688
|
+
sourceRevision: sourceIdentity.sourceRevision ?? undefined,
|
|
1689
|
+
sourceFormat,
|
|
1690
|
+
conversionConfigPath: explicit.conversionConfigPath ?? options.conversionConfigPath ?? undefined,
|
|
1691
|
+
conversionConfigDigest: conversionConfigDigest ?? undefined,
|
|
1692
|
+
weightPackId,
|
|
1693
|
+
weightPackHash,
|
|
1694
|
+
shardSetHash,
|
|
1695
|
+
manifestVariantId,
|
|
1696
|
+
modalitySet,
|
|
1697
|
+
materializationProfile,
|
|
1698
|
+
artifactCompleteness: explicit.artifactCompleteness ?? 'complete',
|
|
1699
|
+
});
|
|
1700
|
+
}
|
|
1701
|
+
|
|
1251
1702
|
|
|
1252
1703
|
export function createManifest(
|
|
1253
1704
|
modelId,
|
|
@@ -1344,6 +1795,8 @@ export function createManifest(
|
|
|
1344
1795
|
modelType: resolvedModelType,
|
|
1345
1796
|
quantization: resolvedQuantization,
|
|
1346
1797
|
quantizationInfo: options.quantizationInfo,
|
|
1798
|
+
...(options.artifactIdentity ? { artifactIdentity: options.artifactIdentity } : {}),
|
|
1799
|
+
...(options.weightsRef ? { weightsRef: options.weightsRef } : {}),
|
|
1347
1800
|
architecture: resolvedArchitecture,
|
|
1348
1801
|
moeConfig,
|
|
1349
1802
|
inference,
|
|
@@ -1394,7 +1847,7 @@ export function createManifest(
|
|
|
1394
1847
|
// Main Converter (uses I/O adapter)
|
|
1395
1848
|
// ============================================================================
|
|
1396
1849
|
|
|
1397
|
-
const MAX_TENSOR_TYPED_ARRAY_BYTES =
|
|
1850
|
+
const MAX_TENSOR_TYPED_ARRAY_BYTES = 0x7fff_ffff;
|
|
1398
1851
|
|
|
1399
1852
|
export async function convertModel(model, io, options = {}) {
|
|
1400
1853
|
const { onProgress, signal } = options;
|
|
@@ -1413,7 +1866,14 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1413
1866
|
if (!modelId) {
|
|
1414
1867
|
throw new Error('Missing modelId for conversion');
|
|
1415
1868
|
}
|
|
1416
|
-
const tensors =
|
|
1869
|
+
const tensors = materializeTiedLmHeadTensor(
|
|
1870
|
+
resolveConversionTensors(model, converterConfig),
|
|
1871
|
+
{
|
|
1872
|
+
inference: options.inference ?? converterConfig?.inference ?? null,
|
|
1873
|
+
quantizationInfo: options.quantizationInfo ?? null,
|
|
1874
|
+
modelType: options.modelType ?? model.modelType ?? 'transformer',
|
|
1875
|
+
}
|
|
1876
|
+
);
|
|
1417
1877
|
if (!Array.isArray(tensors) || tensors.length === 0) {
|
|
1418
1878
|
const textOnly = converterConfig?.output?.textOnly === true;
|
|
1419
1879
|
if (textOnly) {
|
|
@@ -1550,6 +2010,12 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1550
2010
|
}
|
|
1551
2011
|
|
|
1552
2012
|
let emittedChunk = false;
|
|
2013
|
+
// For PLE INT4 per-row quantization, each row-chunk returns its own
|
|
2014
|
+
// per-row F32 scale slice via companionData. Accumulate them across
|
|
2015
|
+
// chunks; after the stream completes, write the concatenated scales
|
|
2016
|
+
// blob and attach sourceTransform.scaleSource pointing at it.
|
|
2017
|
+
const companionChunks = [];
|
|
2018
|
+
let accumulatedSourceTransform = null;
|
|
1553
2019
|
await options.largeTensorTransformer({
|
|
1554
2020
|
tensor,
|
|
1555
2021
|
transformContext,
|
|
@@ -1575,12 +2041,57 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1575
2041
|
}
|
|
1576
2042
|
tensorSize += tensorData.byteLength;
|
|
1577
2043
|
await appendTensorBytes(tensorData, tensorSpans);
|
|
2044
|
+
|
|
2045
|
+
if (result?.companionData instanceof Uint8Array && result.companionData.byteLength > 0) {
|
|
2046
|
+
if (!result.sourceTransform) {
|
|
2047
|
+
throw new Error(
|
|
2048
|
+
`Large tensor chunk returned companionData without sourceTransform for ${tensor.name}.`
|
|
2049
|
+
);
|
|
2050
|
+
}
|
|
2051
|
+
companionChunks.push(result.companionData);
|
|
2052
|
+
if (!accumulatedSourceTransform) {
|
|
2053
|
+
accumulatedSourceTransform = { ...result.sourceTransform };
|
|
2054
|
+
}
|
|
2055
|
+
} else if (result?.sourceTransform && !accumulatedSourceTransform) {
|
|
2056
|
+
accumulatedSourceTransform = { ...result.sourceTransform };
|
|
2057
|
+
}
|
|
1578
2058
|
},
|
|
1579
2059
|
});
|
|
1580
2060
|
|
|
1581
2061
|
if (!emittedChunk) {
|
|
1582
2062
|
throw new Error(`Large tensor transformer did not emit any bytes for ${tensor.name}.`);
|
|
1583
2063
|
}
|
|
2064
|
+
|
|
2065
|
+
if (accumulatedSourceTransform) {
|
|
2066
|
+
let companionBytes = null;
|
|
2067
|
+
if (companionChunks.length > 0) {
|
|
2068
|
+
let totalCompanionBytes = 0;
|
|
2069
|
+
for (const chunk of companionChunks) totalCompanionBytes += chunk.byteLength;
|
|
2070
|
+
companionBytes = new Uint8Array(totalCompanionBytes);
|
|
2071
|
+
let offset = 0;
|
|
2072
|
+
for (const chunk of companionChunks) {
|
|
2073
|
+
companionBytes.set(chunk, offset);
|
|
2074
|
+
offset += chunk.byteLength;
|
|
2075
|
+
}
|
|
2076
|
+
const companionSpans = [];
|
|
2077
|
+
await appendTensorBytes(companionBytes, companionSpans);
|
|
2078
|
+
if (companionSpans.length !== 1) {
|
|
2079
|
+
throw new Error(
|
|
2080
|
+
`Companion scales for ${tensor.name} must land in a single shard (got ${companionSpans.length}).`
|
|
2081
|
+
);
|
|
2082
|
+
}
|
|
2083
|
+
transformContext._pleSourceTransform = {
|
|
2084
|
+
...accumulatedSourceTransform,
|
|
2085
|
+
scaleSource: {
|
|
2086
|
+
shard: companionSpans[0].shardIndex,
|
|
2087
|
+
offset: companionSpans[0].offset,
|
|
2088
|
+
size: companionBytes.byteLength,
|
|
2089
|
+
},
|
|
2090
|
+
};
|
|
2091
|
+
} else {
|
|
2092
|
+
transformContext._pleSourceTransform = accumulatedSourceTransform;
|
|
2093
|
+
}
|
|
2094
|
+
}
|
|
1584
2095
|
} else {
|
|
1585
2096
|
const data = await io.readTensorData(tensor);
|
|
1586
2097
|
const tensorDataInput = new Uint8Array(data);
|
|
@@ -1603,11 +2114,44 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1603
2114
|
outLayout = transformResult?.outLayout ?? null;
|
|
1604
2115
|
tensorSize = tensorData.byteLength;
|
|
1605
2116
|
await appendTensorBytes(tensorData, tensorSpans);
|
|
2117
|
+
|
|
2118
|
+
// Companion data (e.g., INT4 per-row scales for PLE) writes to its own
|
|
2119
|
+
// spans and is referenced from sourceTransform.scaleSource on the
|
|
2120
|
+
// primary tensor location.
|
|
2121
|
+
const companionData = transformResult?.companionData;
|
|
2122
|
+
if (companionData instanceof Uint8Array && companionData.byteLength > 0) {
|
|
2123
|
+
if (!transformResult?.sourceTransform) {
|
|
2124
|
+
throw new Error(
|
|
2125
|
+
`Tensor transformer returned companionData without sourceTransform for ${tensor.name}.`
|
|
2126
|
+
);
|
|
2127
|
+
}
|
|
2128
|
+
const companionSpans = [];
|
|
2129
|
+
await appendTensorBytes(companionData, companionSpans);
|
|
2130
|
+
if (companionSpans.length !== 1) {
|
|
2131
|
+
throw new Error(
|
|
2132
|
+
`Companion scales for ${tensor.name} must land in a single shard (got ${companionSpans.length}).`
|
|
2133
|
+
);
|
|
2134
|
+
}
|
|
2135
|
+
transformContext._pleSourceTransform = {
|
|
2136
|
+
...transformResult.sourceTransform,
|
|
2137
|
+
scaleSource: {
|
|
2138
|
+
shard: companionSpans[0].shardIndex,
|
|
2139
|
+
offset: companionSpans[0].offset,
|
|
2140
|
+
size: companionData.byteLength,
|
|
2141
|
+
},
|
|
2142
|
+
};
|
|
2143
|
+
} else if (transformResult?.sourceTransform) {
|
|
2144
|
+
transformContext._pleSourceTransform = transformResult.sourceTransform;
|
|
2145
|
+
} else {
|
|
2146
|
+
transformContext._pleSourceTransform = null;
|
|
2147
|
+
}
|
|
1606
2148
|
}
|
|
1607
2149
|
|
|
1608
2150
|
// Record tensor location
|
|
1609
2151
|
const role = resolveTensorRole(tensor);
|
|
1610
2152
|
const group = resolveTensorGroup(tensor, tensorGroupModelType);
|
|
2153
|
+
const pleSourceTransform = transformContext._pleSourceTransform ?? null;
|
|
2154
|
+
transformContext._pleSourceTransform = null;
|
|
1611
2155
|
|
|
1612
2156
|
if (tensorSpans.length === 1) {
|
|
1613
2157
|
tensorLocations[tensor.name] = {
|
|
@@ -1619,6 +2163,7 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1619
2163
|
role,
|
|
1620
2164
|
group,
|
|
1621
2165
|
...(outLayout ? { layout: outLayout } : {}),
|
|
2166
|
+
...(pleSourceTransform ? { sourceTransform: pleSourceTransform } : {}),
|
|
1622
2167
|
};
|
|
1623
2168
|
} else {
|
|
1624
2169
|
tensorLocations[tensor.name] = {
|
|
@@ -1629,6 +2174,7 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1629
2174
|
role,
|
|
1630
2175
|
group,
|
|
1631
2176
|
...(outLayout ? { layout: outLayout } : {}),
|
|
2177
|
+
...(pleSourceTransform ? { sourceTransform: pleSourceTransform } : {}),
|
|
1632
2178
|
};
|
|
1633
2179
|
}
|
|
1634
2180
|
|
|
@@ -1662,11 +2208,34 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1662
2208
|
options.quantization ?? model.quantization
|
|
1663
2209
|
);
|
|
1664
2210
|
|
|
2211
|
+
validateInt4PleMaterializationContract(tensorLocations, options.inference, modelId);
|
|
2212
|
+
|
|
2213
|
+
const artifactIdentity = await buildArtifactIdentity({
|
|
2214
|
+
modelId,
|
|
2215
|
+
modelType: options.modelType,
|
|
2216
|
+
source: options.source,
|
|
2217
|
+
sourcePath: options.sourcePath,
|
|
2218
|
+
sourceFormat: options.sourceFormat,
|
|
2219
|
+
conversionConfigPath: options.conversionConfigPath,
|
|
2220
|
+
conversionConfig: options.conversionConfig,
|
|
2221
|
+
explicitArtifactIdentity: converterConfig?.manifest?.artifactIdentity ?? options.artifactIdentity ?? null,
|
|
2222
|
+
hashString: options.hashString,
|
|
2223
|
+
hashAlgorithm: converterConfig.manifest.hashAlgorithm,
|
|
2224
|
+
shards,
|
|
2225
|
+
tensorLocations,
|
|
2226
|
+
quantizationInfo: effectiveQuantizationInfo,
|
|
2227
|
+
inference: options.inference,
|
|
2228
|
+
manifestConfig: converterConfig?.manifest ?? null,
|
|
2229
|
+
converterConfig,
|
|
2230
|
+
});
|
|
2231
|
+
|
|
1665
2232
|
const manifest = createManifest(modelId, model, shards, tensorLocations, {
|
|
1666
|
-
source: 'convert-core',
|
|
2233
|
+
source: options.source ?? 'convert-core',
|
|
1667
2234
|
modelType: options.modelType,
|
|
1668
2235
|
quantization: effectiveManifestQuantization,
|
|
1669
2236
|
quantizationInfo: effectiveQuantizationInfo,
|
|
2237
|
+
artifactIdentity,
|
|
2238
|
+
weightsRef: converterConfig?.manifest?.weightsRef ?? options.weightsRef ?? null,
|
|
1670
2239
|
hashAlgorithm: converterConfig.manifest.hashAlgorithm,
|
|
1671
2240
|
architecture: options.architecture,
|
|
1672
2241
|
inference: options.inference,
|