doppler-gpu 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +67 -34
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.js +5 -1
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/version.js +1 -1
package/src/converter/core.js
CHANGED
|
@@ -170,9 +170,77 @@ export function normalizeStorageQuant(value) {
|
|
|
170
170
|
if (lower === 'fp32' || lower === 'float32') return 'f32';
|
|
171
171
|
if (lower === 'bfloat16') return 'bf16';
|
|
172
172
|
if (lower === 'q4_k_m' || lower === 'q4km') return 'q4k';
|
|
173
|
+
if (lower === 'q4_0' || lower === 'q4-0') return 'q4_0';
|
|
174
|
+
if (
|
|
175
|
+
lower === 'w4a16-ct'
|
|
176
|
+
|| lower === 'w4a16_ct'
|
|
177
|
+
|| lower === 'compressed-tensors-w4a16'
|
|
178
|
+
|| lower === 'compressed_tensors_w4a16'
|
|
179
|
+
) return 'w4a16';
|
|
180
|
+
if (lower === 'wna8-o8' || lower === 'wna8_o8') return 'wna8o8';
|
|
173
181
|
return lower;
|
|
174
182
|
}
|
|
175
183
|
|
|
184
|
+
const SOURCE_PACKED_QUANT_DTYPES = new Set(['q4_0', 'w4a16', 'wna8o8']);
|
|
185
|
+
|
|
186
|
+
const SOURCE_PACKED_MANIFEST_DTYPES = {
|
|
187
|
+
q4_0: 'Q4_0',
|
|
188
|
+
w4a16: 'W4A16',
|
|
189
|
+
wna8o8: 'WNA8O8',
|
|
190
|
+
};
|
|
191
|
+
|
|
192
|
+
const SOURCE_PACKED_STORAGE_DESCRIPTORS = {
|
|
193
|
+
q4_0: {
|
|
194
|
+
packing: 'q4_0',
|
|
195
|
+
blockShape: [32],
|
|
196
|
+
blockBytes: 18,
|
|
197
|
+
},
|
|
198
|
+
w4a16: {
|
|
199
|
+
packing: 'w4a16',
|
|
200
|
+
blockShape: [32],
|
|
201
|
+
blockBytes: 16,
|
|
202
|
+
},
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
const COMPRESSED_TENSORS_W4A16_SUFFIXES = {
|
|
206
|
+
packed: '.weight_packed',
|
|
207
|
+
scale: '.weight_scale',
|
|
208
|
+
shape: '.weight_shape',
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
function cloneSourcePackedStorageDescriptor(targetQuant) {
|
|
212
|
+
const descriptor = SOURCE_PACKED_STORAGE_DESCRIPTORS[targetQuant];
|
|
213
|
+
if (!descriptor) return null;
|
|
214
|
+
return {
|
|
215
|
+
...descriptor,
|
|
216
|
+
blockShape: [...descriptor.blockShape],
|
|
217
|
+
...(Array.isArray(descriptor.companions)
|
|
218
|
+
? { companions: descriptor.companions.map((companion) => ({ ...companion })) }
|
|
219
|
+
: {}),
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function resolveExplicitRoleQuant(tensor, quantizationInfo) {
|
|
224
|
+
if (!quantizationInfo || typeof quantizationInfo !== 'object') {
|
|
225
|
+
return null;
|
|
226
|
+
}
|
|
227
|
+
const role = resolveTensorRole(tensor);
|
|
228
|
+
if (role === 'embedding') {
|
|
229
|
+
return normalizeStorageQuant(quantizationInfo.embeddings ?? null);
|
|
230
|
+
}
|
|
231
|
+
if (role === 'lm_head') {
|
|
232
|
+
return normalizeStorageQuant(
|
|
233
|
+
quantizationInfo.lmHead
|
|
234
|
+
?? quantizationInfo.embeddings
|
|
235
|
+
?? null
|
|
236
|
+
);
|
|
237
|
+
}
|
|
238
|
+
if (role === 'matmul' || role === 'expert' || role === 'router') {
|
|
239
|
+
return normalizeStorageQuant(quantizationInfo.weights ?? null);
|
|
240
|
+
}
|
|
241
|
+
return null;
|
|
242
|
+
}
|
|
243
|
+
|
|
176
244
|
export function resolveTensorTargetQuant(tensorOrName, fallbackQuant, quantizationInfo) {
|
|
177
245
|
const fallback = normalizeStorageQuant(fallbackQuant);
|
|
178
246
|
if (!quantizationInfo || typeof quantizationInfo !== 'object') {
|
|
@@ -214,30 +282,45 @@ function shouldExcludeTextOnlyTensor(name) {
|
|
|
214
282
|
const lower = name.toLowerCase();
|
|
215
283
|
return lower.startsWith('vision_tower.')
|
|
216
284
|
|| lower.startsWith('model.vision_tower.')
|
|
285
|
+
|| lower.startsWith('model.encoder.vision_tower.')
|
|
217
286
|
|| lower.startsWith('vision_model.')
|
|
218
287
|
|| lower.startsWith('model.vision_model.')
|
|
288
|
+
|| lower.startsWith('model.encoder.vision_model.')
|
|
219
289
|
|| lower.startsWith('visual.')
|
|
220
290
|
|| lower.startsWith('model.visual.')
|
|
291
|
+
|| lower.startsWith('model.encoder.visual.')
|
|
221
292
|
|| lower.startsWith('embed_vision.')
|
|
222
293
|
|| lower.startsWith('model.embed_vision.')
|
|
294
|
+
|| lower.startsWith('model.encoder.embed_vision.')
|
|
223
295
|
|| lower.startsWith('vision.')
|
|
224
296
|
|| lower.startsWith('model.vision.')
|
|
297
|
+
|| lower.startsWith('model.encoder.vision.')
|
|
225
298
|
|| lower.startsWith('vision_encoder.')
|
|
299
|
+
|| lower.startsWith('model.encoder.vision_encoder.')
|
|
226
300
|
|| lower.startsWith('image_encoder.')
|
|
301
|
+
|| lower.startsWith('model.encoder.image_encoder.')
|
|
227
302
|
|| lower.startsWith('image_tower.')
|
|
303
|
+
|| lower.startsWith('model.encoder.image_tower.')
|
|
228
304
|
|| lower.startsWith('image.')
|
|
229
305
|
|| lower.startsWith('model.image.')
|
|
306
|
+
|| lower.startsWith('model.encoder.image.')
|
|
230
307
|
|| lower.startsWith('audio_tower.')
|
|
231
308
|
|| lower.startsWith('model.audio_tower.')
|
|
309
|
+
|| lower.startsWith('model.encoder.audio_tower.')
|
|
232
310
|
|| lower.startsWith('audio_model.')
|
|
233
311
|
|| lower.startsWith('model.audio_model.')
|
|
312
|
+
|| lower.startsWith('model.encoder.audio_model.')
|
|
234
313
|
|| lower.startsWith('audio.')
|
|
235
314
|
|| lower.startsWith('model.audio.')
|
|
315
|
+
|| lower.startsWith('model.encoder.audio.')
|
|
236
316
|
|| lower.startsWith('audio_encoder.')
|
|
317
|
+
|| lower.startsWith('model.encoder.audio_encoder.')
|
|
237
318
|
|| lower.startsWith('multi_modal_projector.')
|
|
238
319
|
|| lower.startsWith('model.multi_modal_projector.')
|
|
320
|
+
|| lower.startsWith('model.encoder.multi_modal_projector.')
|
|
239
321
|
|| lower.startsWith('mm_projector.')
|
|
240
|
-
|| lower.startsWith('model.mm_projector.')
|
|
322
|
+
|| lower.startsWith('model.mm_projector.')
|
|
323
|
+
|| lower.startsWith('model.encoder.mm_projector.');
|
|
241
324
|
}
|
|
242
325
|
|
|
243
326
|
function resolveConversionTensors(model, converterConfig) {
|
|
@@ -275,15 +358,212 @@ function resolveConversionTensors(model, converterConfig) {
|
|
|
275
358
|
));
|
|
276
359
|
}
|
|
277
360
|
|
|
361
|
+
function normalizePositiveIntegerShape(value, tensorName) {
|
|
362
|
+
if (!Array.isArray(value) || value.length === 0) {
|
|
363
|
+
throw new Error(`Compressed-tensors W4A16 tensor "${tensorName}" is missing a logical shape.`);
|
|
364
|
+
}
|
|
365
|
+
return value.map((entry, index) => {
|
|
366
|
+
const number = Number(entry);
|
|
367
|
+
if (!Number.isInteger(number) || number <= 0) {
|
|
368
|
+
throw new Error(
|
|
369
|
+
`Compressed-tensors W4A16 tensor "${tensorName}" has invalid shape[${index}]=${JSON.stringify(entry)}.`
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
return number;
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
function resolveCompressedTensorBaseName(name, suffix) {
|
|
377
|
+
return name.endsWith(suffix)
|
|
378
|
+
? name.slice(0, -suffix.length)
|
|
379
|
+
: null;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
function collectCompressedTensorsW4A16(tensors) {
|
|
383
|
+
const groups = new Map();
|
|
384
|
+
const groupFor = (baseName) => {
|
|
385
|
+
const existing = groups.get(baseName);
|
|
386
|
+
if (existing) return existing;
|
|
387
|
+
const created = {
|
|
388
|
+
baseName,
|
|
389
|
+
packed: null,
|
|
390
|
+
scale: null,
|
|
391
|
+
shape: null,
|
|
392
|
+
};
|
|
393
|
+
groups.set(baseName, created);
|
|
394
|
+
return created;
|
|
395
|
+
};
|
|
396
|
+
|
|
397
|
+
for (const tensor of tensors) {
|
|
398
|
+
const name = normalizeTensorName(tensor);
|
|
399
|
+
const packedBase = resolveCompressedTensorBaseName(name, COMPRESSED_TENSORS_W4A16_SUFFIXES.packed);
|
|
400
|
+
if (packedBase) {
|
|
401
|
+
groupFor(packedBase).packed = tensor;
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
404
|
+
const scaleBase = resolveCompressedTensorBaseName(name, COMPRESSED_TENSORS_W4A16_SUFFIXES.scale);
|
|
405
|
+
if (scaleBase) {
|
|
406
|
+
groupFor(scaleBase).scale = tensor;
|
|
407
|
+
continue;
|
|
408
|
+
}
|
|
409
|
+
const shapeBase = resolveCompressedTensorBaseName(name, COMPRESSED_TENSORS_W4A16_SUFFIXES.shape);
|
|
410
|
+
if (shapeBase) {
|
|
411
|
+
groupFor(shapeBase).shape = tensor;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
return groups;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function assertCompressedTensorsW4A16Group(group) {
|
|
419
|
+
const missing = [];
|
|
420
|
+
if (!group.packed) missing.push('weight_packed');
|
|
421
|
+
if (!group.scale) missing.push('weight_scale');
|
|
422
|
+
if (!group.shape) missing.push('weight_shape');
|
|
423
|
+
if (missing.length > 0) {
|
|
424
|
+
throw new Error(
|
|
425
|
+
`Compressed-tensors W4A16 tensor "${group.baseName}.weight" is missing companion tensors: ${missing.join(', ')}.`
|
|
426
|
+
);
|
|
427
|
+
}
|
|
428
|
+
const packedShape = normalizePositiveIntegerShape(group.packed.shape, group.packed.name);
|
|
429
|
+
if (packedShape.length !== 2) {
|
|
430
|
+
throw new Error(
|
|
431
|
+
`Compressed-tensors W4A16 tensor "${group.packed.name}" must be 2D; got shape ${JSON.stringify(packedShape)}.`
|
|
432
|
+
);
|
|
433
|
+
}
|
|
434
|
+
const scaleShape = normalizePositiveIntegerShape(group.scale.shape, group.scale.name);
|
|
435
|
+
if (scaleShape.length !== 2) {
|
|
436
|
+
throw new Error(
|
|
437
|
+
`Compressed-tensors W4A16 scale tensor "${group.scale.name}" must be 2D; got shape ${JSON.stringify(scaleShape)}.`
|
|
438
|
+
);
|
|
439
|
+
}
|
|
440
|
+
normalizePositiveIntegerShape(group.shape.shape, group.shape.name);
|
|
441
|
+
const shapeDtype = String(group.shape.dtype || '').toUpperCase();
|
|
442
|
+
if (shapeDtype !== 'I64' && shapeDtype !== 'I32' && shapeDtype !== 'U32') {
|
|
443
|
+
throw new Error(
|
|
444
|
+
`Compressed-tensors W4A16 shape tensor "${group.shape.name}" must use I64, I32, or U32; got "${group.shape.dtype}".`
|
|
445
|
+
);
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
const COMPRESSED_TENSORS_W4A16_PACKED_VALUES_PER_ELEMENT = {
|
|
450
|
+
U8: 2,
|
|
451
|
+
I8: 2,
|
|
452
|
+
U16: 4,
|
|
453
|
+
I16: 4,
|
|
454
|
+
U32: 8,
|
|
455
|
+
I32: 8,
|
|
456
|
+
};
|
|
457
|
+
|
|
458
|
+
function inferCompressedTensorsW4A16LogicalShape(group) {
|
|
459
|
+
const packedShape = normalizePositiveIntegerShape(group.packed.shape, group.packed.name);
|
|
460
|
+
const packedDtype = String(group.packed.dtype || '').toUpperCase();
|
|
461
|
+
const valuesPerElement = COMPRESSED_TENSORS_W4A16_PACKED_VALUES_PER_ELEMENT[packedDtype];
|
|
462
|
+
if (!valuesPerElement) {
|
|
463
|
+
throw new Error(
|
|
464
|
+
`Compressed-tensors W4A16 tensor "${group.packed.name}" has unsupported packed dtype "${group.packed.dtype}".`
|
|
465
|
+
);
|
|
466
|
+
}
|
|
467
|
+
return [packedShape[0], packedShape[1] * valuesPerElement];
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
function shouldNormalizeCompressedTensorsW4A16(converterConfig) {
|
|
471
|
+
return normalizeStorageQuant(converterConfig?.quantization?.weights) === 'w4a16'
|
|
472
|
+
|| normalizeStorageQuant(converterConfig?.quantization?.sourceQuantizationTarget) === 'w4a16';
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
function sortTensorsForConversion(tensors) {
|
|
476
|
+
return [...tensors].sort((left, right) => normalizeTensorName(left).localeCompare(normalizeTensorName(right)));
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
function normalizeCompressedTensorsW4A16(tensors, converterConfig) {
|
|
480
|
+
const groups = collectCompressedTensorsW4A16(tensors);
|
|
481
|
+
if (groups.size === 0) {
|
|
482
|
+
return tensors;
|
|
483
|
+
}
|
|
484
|
+
if (!shouldNormalizeCompressedTensorsW4A16(converterConfig)) {
|
|
485
|
+
throw new Error(
|
|
486
|
+
'Compressed-tensors W4A16 tensors were detected, but converter.quantization.weights or ' +
|
|
487
|
+
'converter.quantization.sourceQuantizationTarget is not "w4a16".'
|
|
488
|
+
);
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
const byName = new Map(tensors.map((tensor) => [normalizeTensorName(tensor), tensor]));
|
|
492
|
+
const consumed = new Set();
|
|
493
|
+
const companionByName = new Map();
|
|
494
|
+
const synthetic = [];
|
|
495
|
+
const sortedGroups = [...groups.values()].sort((left, right) => left.baseName.localeCompare(right.baseName));
|
|
496
|
+
for (const group of sortedGroups) {
|
|
497
|
+
assertCompressedTensorsW4A16Group(group);
|
|
498
|
+
const logicalName = `${group.baseName}.weight`;
|
|
499
|
+
if (byName.has(logicalName)) {
|
|
500
|
+
throw new Error(
|
|
501
|
+
`Compressed-tensors W4A16 logical tensor "${logicalName}" conflicts with an existing source tensor.`
|
|
502
|
+
);
|
|
503
|
+
}
|
|
504
|
+
consumed.add(group.packed.name);
|
|
505
|
+
companionByName.set(group.scale.name, {
|
|
506
|
+
...group.scale,
|
|
507
|
+
compressedTensorsW4A16Companion: {
|
|
508
|
+
role: 'scales',
|
|
509
|
+
primary: logicalName,
|
|
510
|
+
},
|
|
511
|
+
});
|
|
512
|
+
companionByName.set(group.shape.name, {
|
|
513
|
+
...group.shape,
|
|
514
|
+
compressedTensorsW4A16Companion: {
|
|
515
|
+
role: 'shape',
|
|
516
|
+
primary: logicalName,
|
|
517
|
+
},
|
|
518
|
+
});
|
|
519
|
+
synthetic.push({
|
|
520
|
+
...group.packed,
|
|
521
|
+
name: logicalName,
|
|
522
|
+
dtype: 'W4A16',
|
|
523
|
+
shape: inferCompressedTensorsW4A16LogicalShape(group),
|
|
524
|
+
packedSourceName: group.packed.name,
|
|
525
|
+
compressedTensorsW4A16: {
|
|
526
|
+
packed: group.packed.name,
|
|
527
|
+
scales: group.scale.name,
|
|
528
|
+
shape: group.shape.name,
|
|
529
|
+
},
|
|
530
|
+
storage: {
|
|
531
|
+
packing: 'w4a16',
|
|
532
|
+
blockShape: [32],
|
|
533
|
+
blockBytes: 16,
|
|
534
|
+
companions: [
|
|
535
|
+
{ role: 'scales', tensorId: group.scale.name },
|
|
536
|
+
{ role: 'shape', tensorId: group.shape.name },
|
|
537
|
+
],
|
|
538
|
+
},
|
|
539
|
+
});
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
return sortTensorsForConversion([
|
|
543
|
+
...tensors
|
|
544
|
+
.filter((tensor) => !consumed.has(normalizeTensorName(tensor)))
|
|
545
|
+
.map((tensor) => companionByName.get(normalizeTensorName(tensor)) ?? tensor),
|
|
546
|
+
...synthetic,
|
|
547
|
+
]);
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
function isCompressedTensorsW4A16CompanionTensor(tensor) {
|
|
551
|
+
return Boolean(
|
|
552
|
+
tensor?.compressedTensorsW4A16Companion
|
|
553
|
+
&& typeof tensor.compressedTensorsW4A16Companion === 'object'
|
|
554
|
+
);
|
|
555
|
+
}
|
|
556
|
+
|
|
278
557
|
function shouldMaterializeTiedLmHead(tensors, options) {
|
|
279
558
|
if (options?.inference?.output?.tieWordEmbeddings !== true) {
|
|
280
559
|
return false;
|
|
281
560
|
}
|
|
282
|
-
|
|
561
|
+
const lmHeadQuant = normalizeStorageQuant(options?.quantizationInfo?.lmHead ?? null);
|
|
562
|
+
if (!SOURCE_PACKED_QUANT_DTYPES.has(lmHeadQuant) && lmHeadQuant !== 'q4k') {
|
|
283
563
|
return false;
|
|
284
564
|
}
|
|
285
565
|
const embeddingQuant = normalizeStorageQuant(options?.quantizationInfo?.embeddings ?? null);
|
|
286
|
-
if (embeddingQuant ===
|
|
566
|
+
if (embeddingQuant === lmHeadQuant) {
|
|
287
567
|
return false;
|
|
288
568
|
}
|
|
289
569
|
return !tensors.some((tensor) => resolveTensorRole(tensor) === 'lm_head');
|
|
@@ -300,6 +580,15 @@ function resolveTiedLmHeadName(embeddingName) {
|
|
|
300
580
|
if (name === 'model.language_model.embed_tokens.weight') {
|
|
301
581
|
return 'model.language_model.lm_head.weight';
|
|
302
582
|
}
|
|
583
|
+
if (name === 'model.decoder.embed_tokens.weight') {
|
|
584
|
+
return 'lm_head.weight';
|
|
585
|
+
}
|
|
586
|
+
if (name === 'decoder.embed_tokens.weight') {
|
|
587
|
+
return 'lm_head.weight';
|
|
588
|
+
}
|
|
589
|
+
if (name === 'model.encoder.language_model.embed_tokens.weight') {
|
|
590
|
+
return 'lm_head.weight';
|
|
591
|
+
}
|
|
303
592
|
if (name === 'language_model.embed_tokens.weight') {
|
|
304
593
|
return 'language_model.lm_head.weight';
|
|
305
594
|
}
|
|
@@ -536,6 +825,9 @@ function resolveMoEExpertFormat(rawConfig, resolvedModelType, quantizationInfo,
|
|
|
536
825
|
if (modelType.includes('gpt_oss') || modelType.includes('gpt-oss') || modelType.includes('gptoss')) {
|
|
537
826
|
return 'gpt-oss';
|
|
538
827
|
}
|
|
828
|
+
if (modelType === 'diffusion_gemma' || modelType === 'diffusion_gemma_text') {
|
|
829
|
+
return 'gemma4';
|
|
830
|
+
}
|
|
539
831
|
return 'mixtral';
|
|
540
832
|
}
|
|
541
833
|
|
|
@@ -544,7 +836,10 @@ function normalizeMoEConfig(config, contextLabel) {
|
|
|
544
836
|
const numExperts = Number(config.numExperts);
|
|
545
837
|
const numExpertsPerToken = Number(config.numExpertsPerToken);
|
|
546
838
|
const expertFormat = String(config.expertFormat || '').trim();
|
|
547
|
-
const
|
|
839
|
+
const expertIntermediateSize = config.expertIntermediateSize == null
|
|
840
|
+
? null
|
|
841
|
+
: Number(config.expertIntermediateSize);
|
|
842
|
+
const allowedExpertFormat = expertFormat === 'gpt-oss' || expertFormat === 'mixtral' || expertFormat === 'gemma4';
|
|
548
843
|
if (!Number.isFinite(numExperts) || numExperts <= 0) {
|
|
549
844
|
throw new Error(`Invalid moeConfig.numExperts for ${contextLabel}`);
|
|
550
845
|
}
|
|
@@ -557,10 +852,20 @@ function normalizeMoEConfig(config, contextLabel) {
|
|
|
557
852
|
if (!allowedExpertFormat) {
|
|
558
853
|
throw new Error(`Invalid moeConfig.expertFormat for ${contextLabel}: "${expertFormat}"`);
|
|
559
854
|
}
|
|
855
|
+
if (
|
|
856
|
+
expertIntermediateSize != null
|
|
857
|
+
&& (!Number.isFinite(expertIntermediateSize) || expertIntermediateSize <= 0)
|
|
858
|
+
) {
|
|
859
|
+
throw new Error(`Invalid moeConfig.expertIntermediateSize for ${contextLabel}`);
|
|
860
|
+
}
|
|
861
|
+
if (expertFormat === 'gemma4' && expertIntermediateSize == null) {
|
|
862
|
+
throw new Error(`Invalid moeConfig for ${contextLabel}: gemma4 experts require expertIntermediateSize`);
|
|
863
|
+
}
|
|
560
864
|
return {
|
|
561
865
|
numExperts,
|
|
562
866
|
numExpertsPerToken,
|
|
563
867
|
expertFormat,
|
|
868
|
+
...(expertIntermediateSize == null ? {} : { expertIntermediateSize }),
|
|
564
869
|
};
|
|
565
870
|
}
|
|
566
871
|
|
|
@@ -605,9 +910,17 @@ export function resolveManifestMoEConfig(model, options, rawConfig, resolvedMode
|
|
|
605
910
|
options?.quantizationInfo ?? null,
|
|
606
911
|
null
|
|
607
912
|
);
|
|
913
|
+
const expertIntermediateSize = expertFormat === 'gemma4'
|
|
914
|
+
? resolveMoEConfigNumber(rawConfig, 'moe_intermediate_size', 'expert_intermediate_size')
|
|
915
|
+
: null;
|
|
608
916
|
|
|
609
917
|
return normalizeMoEConfig(
|
|
610
|
-
{
|
|
918
|
+
{
|
|
919
|
+
numExperts,
|
|
920
|
+
numExpertsPerToken,
|
|
921
|
+
expertFormat,
|
|
922
|
+
...(expertIntermediateSize == null ? {} : { expertIntermediateSize }),
|
|
923
|
+
},
|
|
611
924
|
options?.modelId ?? 'model'
|
|
612
925
|
);
|
|
613
926
|
}
|
|
@@ -978,11 +1291,74 @@ export function transformTensorBytes(tensor, rawData, options = {}) {
|
|
|
978
1291
|
return buildInt4PerRowPleTransform(tensor, tensorDataInput, sourceDtype, options);
|
|
979
1292
|
}
|
|
980
1293
|
|
|
1294
|
+
if (isCompressedTensorsW4A16CompanionTensor(tensor)) {
|
|
1295
|
+
return {
|
|
1296
|
+
tensorData,
|
|
1297
|
+
outDtype: sourceDtype,
|
|
1298
|
+
outLayout: null,
|
|
1299
|
+
sourceDtype,
|
|
1300
|
+
tensorTargetQuant: null,
|
|
1301
|
+
};
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1304
|
+
if (SOURCE_PACKED_QUANT_DTYPES.has(tensorTargetQuant)) {
|
|
1305
|
+
const sourceQuant = normalizeStorageQuant(sourceDtype);
|
|
1306
|
+
if (sourceQuant === tensorTargetQuant) {
|
|
1307
|
+
const descriptor = cloneSourcePackedStorageDescriptor(tensorTargetQuant);
|
|
1308
|
+
const sourceCompanions = Array.isArray(tensor?.storage?.companions)
|
|
1309
|
+
? tensor.storage.companions.map((companion) => ({ ...companion }))
|
|
1310
|
+
: null;
|
|
1311
|
+
return {
|
|
1312
|
+
tensorData,
|
|
1313
|
+
outDtype: SOURCE_PACKED_MANIFEST_DTYPES[tensorTargetQuant],
|
|
1314
|
+
outLayout: null,
|
|
1315
|
+
sourceDtype,
|
|
1316
|
+
tensorTargetQuant,
|
|
1317
|
+
storage: {
|
|
1318
|
+
...descriptor,
|
|
1319
|
+
...(sourceCompanions ? { companions: sourceCompanions } : {}),
|
|
1320
|
+
},
|
|
1321
|
+
};
|
|
1322
|
+
}
|
|
1323
|
+
const roleQuant = resolveExplicitRoleQuant(tensor, quantizationInfo);
|
|
1324
|
+
const requiresPackedSource = (
|
|
1325
|
+
forceQuantizeDecision
|
|
1326
|
+
?? (
|
|
1327
|
+
roleQuant === tensorTargetQuant
|
|
1328
|
+
|| shouldQuantize(tensor.name, tensor.shape, {
|
|
1329
|
+
quantizeEmbeddings,
|
|
1330
|
+
modulesToNotConvert,
|
|
1331
|
+
role: tensor.role ?? null,
|
|
1332
|
+
})
|
|
1333
|
+
)
|
|
1334
|
+
);
|
|
1335
|
+
if (!requiresPackedSource) {
|
|
1336
|
+
return {
|
|
1337
|
+
tensorData,
|
|
1338
|
+
outDtype: sourceDtype,
|
|
1339
|
+
outLayout: null,
|
|
1340
|
+
sourceDtype,
|
|
1341
|
+
tensorTargetQuant,
|
|
1342
|
+
};
|
|
1343
|
+
}
|
|
1344
|
+
throw new Error(
|
|
1345
|
+
`Cannot materialize ${tensorTargetQuant} for ${tensor.name}: ` +
|
|
1346
|
+
`native import requires source dtype ${SOURCE_PACKED_MANIFEST_DTYPES[tensorTargetQuant]}; ` +
|
|
1347
|
+
'the converter does not re-quantize tensors into this packed format.'
|
|
1348
|
+
);
|
|
1349
|
+
}
|
|
1350
|
+
|
|
981
1351
|
if (tensorTargetQuant === 'q4k') {
|
|
982
1352
|
const sourceQuant = normalizeStorageQuant(sourceDtype);
|
|
1353
|
+
const tensorRole = resolveTensorRole(tensor);
|
|
1354
|
+
const isMatrixLikeShape = Array.isArray(tensor.shape) && tensor.shape.length >= 2;
|
|
1355
|
+
const is2DMatrixShape = Array.isArray(tensor.shape) && tensor.shape.length === 2;
|
|
1356
|
+
const useQ4KRowWise = isMatrixLikeShape
|
|
1357
|
+
&& q4kLayout === 'row'
|
|
1358
|
+
&& (is2DMatrixShape || tensorRole === 'expert');
|
|
983
1359
|
if (sourceQuant === 'q4k') {
|
|
984
1360
|
outDtype = 'Q4_K_M';
|
|
985
|
-
if (
|
|
1361
|
+
if (is2DMatrixShape) {
|
|
986
1362
|
outLayout = q4kLayout;
|
|
987
1363
|
}
|
|
988
1364
|
return {
|
|
@@ -1004,15 +1380,17 @@ export function transformTensorBytes(tensor, rawData, options = {}) {
|
|
|
1004
1380
|
if (shouldQuantizeTensor) {
|
|
1005
1381
|
const f32Data = toFloat32ForQ4K(tensorData, sourceDtype, tensor.name);
|
|
1006
1382
|
const quantized = (
|
|
1007
|
-
|
|
1383
|
+
is2DMatrixShape
|
|
1008
1384
|
? (q4kLayout === 'col'
|
|
1009
1385
|
? quantizeToQ4KMColumnWise(f32Data, tensor.shape)
|
|
1010
1386
|
: quantizeToQ4KMRowWise(f32Data, tensor.shape))
|
|
1387
|
+
: useQ4KRowWise
|
|
1388
|
+
? quantizeToQ4KMRowWise(f32Data, tensor.shape)
|
|
1011
1389
|
: quantizeToQ4KM(f32Data, tensor.shape)
|
|
1012
1390
|
);
|
|
1013
1391
|
tensorData = quantized.quantized;
|
|
1014
1392
|
outDtype = 'Q4_K_M';
|
|
1015
|
-
if (
|
|
1393
|
+
if (is2DMatrixShape || useQ4KRowWise) {
|
|
1016
1394
|
outLayout = q4kLayout;
|
|
1017
1395
|
}
|
|
1018
1396
|
} else if (sourceDtype === 'BF16') {
|
|
@@ -1333,12 +1711,37 @@ function getNestedTextConfig(config) {
|
|
|
1333
1711
|
function resolveGemma4TextConfig(rawConfig) {
|
|
1334
1712
|
const textConfig = getNestedTextConfig(rawConfig);
|
|
1335
1713
|
const modelType = String(textConfig?.model_type ?? rawConfig?.model_type ?? '').trim().toLowerCase();
|
|
1336
|
-
if (
|
|
1714
|
+
if (
|
|
1715
|
+
modelType !== 'gemma4'
|
|
1716
|
+
&& modelType !== 'gemma4_text'
|
|
1717
|
+
&& modelType !== 'gemma4_unified'
|
|
1718
|
+
&& modelType !== 'gemma4_unified_text'
|
|
1719
|
+
) {
|
|
1337
1720
|
return null;
|
|
1338
1721
|
}
|
|
1339
1722
|
return textConfig ?? rawConfig ?? null;
|
|
1340
1723
|
}
|
|
1341
1724
|
|
|
1725
|
+
function resolveDiffusionGemmaConfig(rawConfig, resolvedModelType = null) {
|
|
1726
|
+
const textConfig = getNestedTextConfig(rawConfig);
|
|
1727
|
+
const modelType = String(
|
|
1728
|
+
resolvedModelType
|
|
1729
|
+
?? rawConfig?.model_type
|
|
1730
|
+
?? textConfig?.model_type
|
|
1731
|
+
?? ''
|
|
1732
|
+
).trim().toLowerCase();
|
|
1733
|
+
const textModelType = String(textConfig?.model_type ?? '').trim().toLowerCase();
|
|
1734
|
+
if (
|
|
1735
|
+
modelType !== 'diffusion_gemma'
|
|
1736
|
+
&& modelType !== 'diffusion_gemma_text'
|
|
1737
|
+
&& textModelType !== 'diffusion_gemma'
|
|
1738
|
+
&& textModelType !== 'diffusion_gemma_text'
|
|
1739
|
+
) {
|
|
1740
|
+
return null;
|
|
1741
|
+
}
|
|
1742
|
+
return textConfig ?? rawConfig ?? {};
|
|
1743
|
+
}
|
|
1744
|
+
|
|
1342
1745
|
function collectGemma4UnsupportedTensorFlags(tensors) {
|
|
1343
1746
|
const names = Array.isArray(tensors) ? tensors.map((tensor) => normalizeTensorName(tensor).toLowerCase()) : [];
|
|
1344
1747
|
const flags = [];
|
|
@@ -1410,6 +1813,143 @@ function assertSupportedGemma4Conversion(model, tensors, modelId) {
|
|
|
1410
1813
|
);
|
|
1411
1814
|
}
|
|
1412
1815
|
|
|
1816
|
+
function readDiffusionGemmaPositiveInteger(value, label, modelId) {
|
|
1817
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
1818
|
+
throw new Error(`DiffusionGemma model "${modelId}" requires ${label} to be a positive integer.`);
|
|
1819
|
+
}
|
|
1820
|
+
return value;
|
|
1821
|
+
}
|
|
1822
|
+
|
|
1823
|
+
function readDiffusionGemmaNonNegativeNumber(value, label, modelId) {
|
|
1824
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
|
1825
|
+
throw new Error(`DiffusionGemma model "${modelId}" requires ${label} to be a non-negative finite number.`);
|
|
1826
|
+
}
|
|
1827
|
+
return value;
|
|
1828
|
+
}
|
|
1829
|
+
|
|
1830
|
+
function readDiffusionGemmaEntropyBound(generationConfig, modelId) {
|
|
1831
|
+
if (generationConfig && Object.hasOwn(generationConfig, 'entropy_bound')) {
|
|
1832
|
+
return readDiffusionGemmaNonNegativeNumber(
|
|
1833
|
+
generationConfig.entropy_bound,
|
|
1834
|
+
'generation_config.entropy_bound',
|
|
1835
|
+
modelId
|
|
1836
|
+
);
|
|
1837
|
+
}
|
|
1838
|
+
return readDiffusionGemmaNonNegativeNumber(
|
|
1839
|
+
generationConfig?.sampler_config?.entropy_bound,
|
|
1840
|
+
'generation_config.sampler_config.entropy_bound',
|
|
1841
|
+
modelId
|
|
1842
|
+
);
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
function readDiffusionGemmaNullableTokenId(value, label, modelId) {
|
|
1846
|
+
if (value == null) return null;
|
|
1847
|
+
if (!Number.isInteger(value) || value < 0) {
|
|
1848
|
+
throw new Error(`DiffusionGemma model "${modelId}" requires ${label} to be null or a non-negative integer.`);
|
|
1849
|
+
}
|
|
1850
|
+
return value;
|
|
1851
|
+
}
|
|
1852
|
+
|
|
1853
|
+
function readDiffusionGemmaTokenId(value, label, modelId) {
|
|
1854
|
+
if (!Number.isInteger(value) || value < 0) {
|
|
1855
|
+
throw new Error(`DiffusionGemma model "${modelId}" requires ${label} to be a non-negative integer.`);
|
|
1856
|
+
}
|
|
1857
|
+
return value;
|
|
1858
|
+
}
|
|
1859
|
+
|
|
1860
|
+
function readDiffusionGemmaEosTokenIds(rawConfig, generationConfig, modelId) {
|
|
1861
|
+
const eos = resolveTokenizerIds(
|
|
1862
|
+
generationConfig?.eos_token_id
|
|
1863
|
+
?? generationConfig?.eos_token_ids
|
|
1864
|
+
?? rawConfig?.eos_token_id
|
|
1865
|
+
?? rawConfig?.eos_token_ids
|
|
1866
|
+
);
|
|
1867
|
+
if (!Array.isArray(eos) || eos.length === 0) {
|
|
1868
|
+
throw new Error(`DiffusionGemma model "${modelId}" requires generation_config.eos_token_id.`);
|
|
1869
|
+
}
|
|
1870
|
+
return eos;
|
|
1871
|
+
}
|
|
1872
|
+
|
|
1873
|
+
function resolveDiffusionGemmaInferenceContract(rawConfig, generationConfig, modelId) {
|
|
1874
|
+
const canvasLength = readDiffusionGemmaPositiveInteger(
|
|
1875
|
+
rawConfig?.canvas_length,
|
|
1876
|
+
'config.canvas_length',
|
|
1877
|
+
modelId
|
|
1878
|
+
);
|
|
1879
|
+
const maxDenoisingSteps = readDiffusionGemmaPositiveInteger(
|
|
1880
|
+
generationConfig?.max_denoising_steps,
|
|
1881
|
+
'generation_config.max_denoising_steps',
|
|
1882
|
+
modelId
|
|
1883
|
+
);
|
|
1884
|
+
const maxNewTokens = readDiffusionGemmaPositiveInteger(
|
|
1885
|
+
generationConfig?.max_new_tokens,
|
|
1886
|
+
'generation_config.max_new_tokens',
|
|
1887
|
+
modelId
|
|
1888
|
+
);
|
|
1889
|
+
const stabilityThreshold = readDiffusionGemmaPositiveInteger(
|
|
1890
|
+
generationConfig?.stability_threshold,
|
|
1891
|
+
'generation_config.stability_threshold',
|
|
1892
|
+
modelId
|
|
1893
|
+
);
|
|
1894
|
+
const padTokenId = readDiffusionGemmaTokenId(
|
|
1895
|
+
generationConfig?.pad_token_id ?? rawConfig?.pad_token_id,
|
|
1896
|
+
'generation_config.pad_token_id',
|
|
1897
|
+
modelId
|
|
1898
|
+
);
|
|
1899
|
+
const tMin = readDiffusionGemmaNonNegativeNumber(
|
|
1900
|
+
generationConfig?.t_min,
|
|
1901
|
+
'generation_config.t_min',
|
|
1902
|
+
modelId
|
|
1903
|
+
);
|
|
1904
|
+
const tMax = readDiffusionGemmaNonNegativeNumber(
|
|
1905
|
+
generationConfig?.t_max,
|
|
1906
|
+
'generation_config.t_max',
|
|
1907
|
+
modelId
|
|
1908
|
+
);
|
|
1909
|
+
if (tMax < tMin) {
|
|
1910
|
+
throw new Error(`DiffusionGemma model "${modelId}" requires generation_config.t_max >= t_min.`);
|
|
1911
|
+
}
|
|
1912
|
+
return {
|
|
1913
|
+
canvasLength,
|
|
1914
|
+
maxDenoisingSteps,
|
|
1915
|
+
maxNewTokens,
|
|
1916
|
+
tMin,
|
|
1917
|
+
tMax,
|
|
1918
|
+
entropyBound: readDiffusionGemmaEntropyBound(generationConfig, modelId),
|
|
1919
|
+
confidenceThreshold: readDiffusionGemmaNonNegativeNumber(
|
|
1920
|
+
generationConfig?.confidence_threshold,
|
|
1921
|
+
'generation_config.confidence_threshold',
|
|
1922
|
+
modelId
|
|
1923
|
+
),
|
|
1924
|
+
stabilityThreshold,
|
|
1925
|
+
padTokenId,
|
|
1926
|
+
eosTokenIds: readDiffusionGemmaEosTokenIds(rawConfig, generationConfig, modelId),
|
|
1927
|
+
boiTokenId: readDiffusionGemmaNullableTokenId(rawConfig?.boi_token_id, 'config.boi_token_id', modelId),
|
|
1928
|
+
eoiTokenId: readDiffusionGemmaNullableTokenId(rawConfig?.eoi_token_id, 'config.eoi_token_id', modelId),
|
|
1929
|
+
imageTokenId: readDiffusionGemmaNullableTokenId(rawConfig?.image_token_id, 'config.image_token_id', modelId),
|
|
1930
|
+
selfConditioning: true,
|
|
1931
|
+
decoderCacheMode: 'encoder_kv_readonly_canvas_concat',
|
|
1932
|
+
router: {
|
|
1933
|
+
scaleHiddenStates: true,
|
|
1934
|
+
normalizeTopK: true,
|
|
1935
|
+
perExpertScale: true,
|
|
1936
|
+
},
|
|
1937
|
+
};
|
|
1938
|
+
}
|
|
1939
|
+
|
|
1940
|
+
function applyDiffusionGemmaInferenceContract(inference, rawConfig, generationConfig, modelId, resolvedModelType) {
|
|
1941
|
+
if (!resolveDiffusionGemmaConfig(rawConfig, resolvedModelType)) {
|
|
1942
|
+
return inference;
|
|
1943
|
+
}
|
|
1944
|
+
if (inference?.diffusionGemma && typeof inference.diffusionGemma === 'object') {
|
|
1945
|
+
return inference;
|
|
1946
|
+
}
|
|
1947
|
+
return {
|
|
1948
|
+
...inference,
|
|
1949
|
+
diffusionGemma: resolveDiffusionGemmaInferenceContract(rawConfig, generationConfig, modelId),
|
|
1950
|
+
};
|
|
1951
|
+
}
|
|
1952
|
+
|
|
1413
1953
|
|
|
1414
1954
|
export function buildTensorMap(tensors, shardSize) {
|
|
1415
1955
|
if (!shardSize || shardSize <= 0) {
|
|
@@ -1740,6 +2280,13 @@ export function createManifest(
|
|
|
1740
2280
|
if (!inference) {
|
|
1741
2281
|
throw new Error('inference config is required — use a v1 conversion config');
|
|
1742
2282
|
}
|
|
2283
|
+
inference = applyDiffusionGemmaInferenceContract(
|
|
2284
|
+
inference,
|
|
2285
|
+
rawConfig,
|
|
2286
|
+
generationConfig,
|
|
2287
|
+
modelId,
|
|
2288
|
+
resolvedModelType
|
|
2289
|
+
);
|
|
1743
2290
|
|
|
1744
2291
|
const embeddingOutput = inferEmbeddingOutputConfig(tensorLocations);
|
|
1745
2292
|
const hasExplicitEmbeddingPostprocessor = Object.prototype.hasOwnProperty.call(
|
|
@@ -1779,7 +2326,8 @@ export function createManifest(
|
|
|
1779
2326
|
throw new Error('Missing hashAlgorithm for manifest');
|
|
1780
2327
|
}
|
|
1781
2328
|
|
|
1782
|
-
const
|
|
2329
|
+
const isTextOnlyArtifact = options.textOnly === true;
|
|
2330
|
+
const multimodalConfig = isDiffusion || isTextOnlyArtifact
|
|
1783
2331
|
? { vision_config: null, audio_config: null }
|
|
1784
2332
|
: resolveManifestMultimodalConfig(rawConfig, manifestPolicy);
|
|
1785
2333
|
const manifestConfig = isDiffusion
|
|
@@ -1867,7 +2415,10 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1867
2415
|
throw new Error('Missing modelId for conversion');
|
|
1868
2416
|
}
|
|
1869
2417
|
const tensors = materializeTiedLmHeadTensor(
|
|
1870
|
-
|
|
2418
|
+
normalizeCompressedTensorsW4A16(
|
|
2419
|
+
resolveConversionTensors(model, converterConfig),
|
|
2420
|
+
converterConfig
|
|
2421
|
+
),
|
|
1871
2422
|
{
|
|
1872
2423
|
inference: options.inference ?? converterConfig?.inference ?? null,
|
|
1873
2424
|
quantizationInfo: options.quantizationInfo ?? null,
|
|
@@ -1996,6 +2547,7 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1996
2547
|
const sourceTensorSize = Number.isFinite(tensor?.size) ? Number(tensor.size) : null;
|
|
1997
2548
|
let outDtype = tensor.dtype;
|
|
1998
2549
|
let outLayout = null;
|
|
2550
|
+
let tensorStorage = null;
|
|
1999
2551
|
let tensorSize = 0;
|
|
2000
2552
|
|
|
2001
2553
|
if (
|
|
@@ -2027,9 +2579,11 @@ export async function convertModel(model, io, options = {}) {
|
|
|
2027
2579
|
}
|
|
2028
2580
|
const chunkOutDtype = result?.outDtype ?? tensor.dtype;
|
|
2029
2581
|
const chunkOutLayout = result?.outLayout ?? null;
|
|
2582
|
+
const chunkStorage = result?.storage ?? null;
|
|
2030
2583
|
if (!emittedChunk) {
|
|
2031
2584
|
outDtype = chunkOutDtype;
|
|
2032
2585
|
outLayout = chunkOutLayout;
|
|
2586
|
+
tensorStorage = chunkStorage;
|
|
2033
2587
|
emittedChunk = true;
|
|
2034
2588
|
} else {
|
|
2035
2589
|
if (chunkOutDtype !== outDtype) {
|
|
@@ -2038,6 +2592,9 @@ export async function convertModel(model, io, options = {}) {
|
|
|
2038
2592
|
if (chunkOutLayout !== outLayout) {
|
|
2039
2593
|
throw new Error(`Large tensor transformer returned inconsistent layout for ${tensor.name}.`);
|
|
2040
2594
|
}
|
|
2595
|
+
if (JSON.stringify(chunkStorage) !== JSON.stringify(tensorStorage)) {
|
|
2596
|
+
throw new Error(`Large tensor transformer returned inconsistent storage descriptor for ${tensor.name}.`);
|
|
2597
|
+
}
|
|
2041
2598
|
}
|
|
2042
2599
|
tensorSize += tensorData.byteLength;
|
|
2043
2600
|
await appendTensorBytes(tensorData, tensorSpans);
|
|
@@ -2112,6 +2669,7 @@ export async function convertModel(model, io, options = {}) {
|
|
|
2112
2669
|
}
|
|
2113
2670
|
outDtype = transformResult?.outDtype ?? tensor.dtype;
|
|
2114
2671
|
outLayout = transformResult?.outLayout ?? null;
|
|
2672
|
+
tensorStorage = transformResult?.storage ?? null;
|
|
2115
2673
|
tensorSize = tensorData.byteLength;
|
|
2116
2674
|
await appendTensorBytes(tensorData, tensorSpans);
|
|
2117
2675
|
|
|
@@ -2163,6 +2721,7 @@ export async function convertModel(model, io, options = {}) {
|
|
|
2163
2721
|
role,
|
|
2164
2722
|
group,
|
|
2165
2723
|
...(outLayout ? { layout: outLayout } : {}),
|
|
2724
|
+
...(tensorStorage ? { storage: tensorStorage } : {}),
|
|
2166
2725
|
...(pleSourceTransform ? { sourceTransform: pleSourceTransform } : {}),
|
|
2167
2726
|
};
|
|
2168
2727
|
} else {
|
|
@@ -2174,6 +2733,7 @@ export async function convertModel(model, io, options = {}) {
|
|
|
2174
2733
|
role,
|
|
2175
2734
|
group,
|
|
2176
2735
|
...(outLayout ? { layout: outLayout } : {}),
|
|
2736
|
+
...(tensorStorage ? { storage: tensorStorage } : {}),
|
|
2177
2737
|
...(pleSourceTransform ? { sourceTransform: pleSourceTransform } : {}),
|
|
2178
2738
|
};
|
|
2179
2739
|
}
|
|
@@ -2234,15 +2794,17 @@ export async function convertModel(model, io, options = {}) {
|
|
|
2234
2794
|
modelType: options.modelType,
|
|
2235
2795
|
quantization: effectiveManifestQuantization,
|
|
2236
2796
|
quantizationInfo: effectiveQuantizationInfo,
|
|
2797
|
+
moeConfig: converterConfig?.moeConfig ?? options.moeConfig ?? null,
|
|
2237
2798
|
artifactIdentity,
|
|
2238
2799
|
weightsRef: converterConfig?.manifest?.weightsRef ?? options.weightsRef ?? null,
|
|
2239
2800
|
hashAlgorithm: converterConfig.manifest.hashAlgorithm,
|
|
2240
2801
|
architecture: options.architecture,
|
|
2241
2802
|
inference: options.inference,
|
|
2242
|
-
eosTokenId: options.eosTokenId,
|
|
2803
|
+
eosTokenId: converterConfig?.manifest?.eosTokenId ?? options.eosTokenId,
|
|
2243
2804
|
convertedAt: converterConfig?.manifest?.conversion?.convertedAt ?? null,
|
|
2244
2805
|
conversionInfo: converterConfig?.manifest?.conversion ?? null,
|
|
2245
2806
|
manifestConfig: converterConfig?.manifest ?? null,
|
|
2807
|
+
textOnly: converterConfig?.output?.textOnly === true,
|
|
2246
2808
|
});
|
|
2247
2809
|
|
|
2248
2810
|
// Write manifest
|