doppler-gpu 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -2,6 +2,7 @@ import {
|
|
|
2
2
|
EXECUTION_V1_SCHEMA_ID,
|
|
3
3
|
expandExecutionV1,
|
|
4
4
|
} from '../config/schema/index.js';
|
|
5
|
+
import { validateRequiredInferenceFields } from '../inference/pipelines/text/config.js';
|
|
5
6
|
import {
|
|
6
7
|
buildQuantizationInfo,
|
|
7
8
|
resolveManifestQuantization,
|
|
@@ -101,6 +102,13 @@ function isV1Config(converterConfig) {
|
|
|
101
102
|
return exec && typeof exec === 'object' && exec.kernels && typeof exec.kernels === 'object';
|
|
102
103
|
}
|
|
103
104
|
|
|
105
|
+
function cloneJsonValue(value) {
|
|
106
|
+
if (value == null) return value;
|
|
107
|
+
if (typeof structuredClone === 'function') {
|
|
108
|
+
return structuredClone(value);
|
|
109
|
+
}
|
|
110
|
+
return JSON.parse(JSON.stringify(value));
|
|
111
|
+
}
|
|
104
112
|
|
|
105
113
|
function validateV1InferenceFields(inference, modelId) {
|
|
106
114
|
const required = ['attention', 'normalization', 'ffn', 'rope', 'output', 'chatTemplate'];
|
|
@@ -112,6 +120,10 @@ function validateV1InferenceFields(inference, modelId) {
|
|
|
112
120
|
);
|
|
113
121
|
}
|
|
114
122
|
}
|
|
123
|
+
validateRequiredInferenceFields(
|
|
124
|
+
cloneJsonValue(inference),
|
|
125
|
+
modelId
|
|
126
|
+
);
|
|
115
127
|
}
|
|
116
128
|
|
|
117
129
|
|
|
@@ -121,7 +133,9 @@ function resolveConversionPlanV1(options) {
|
|
|
121
133
|
const converterConfig = options.converterConfig;
|
|
122
134
|
const inference = converterConfig.inference;
|
|
123
135
|
const execution = converterConfig.execution;
|
|
124
|
-
const
|
|
136
|
+
const session = converterConfig.session;
|
|
137
|
+
const modelType = converterConfig?.modelType ?? rawConfig?.model_type ?? 'transformer';
|
|
138
|
+
const requiresSessionPolicy = modelType !== 'embedding';
|
|
125
139
|
|
|
126
140
|
if (!inference || typeof inference !== 'object') {
|
|
127
141
|
throw new Error(
|
|
@@ -133,9 +147,9 @@ function resolveConversionPlanV1(options) {
|
|
|
133
147
|
'V1 config requires execution with kernels, decode, and prefill arrays.'
|
|
134
148
|
);
|
|
135
149
|
}
|
|
136
|
-
if (!
|
|
150
|
+
if (requiresSessionPolicy && (!session || typeof session !== 'object')) {
|
|
137
151
|
throw new Error(
|
|
138
|
-
'V1 config requires
|
|
152
|
+
'V1 config requires session with compute defaults and kvcache policy.'
|
|
139
153
|
);
|
|
140
154
|
}
|
|
141
155
|
if (!execution.policies || typeof execution.policies !== 'object') {
|
|
@@ -158,9 +172,9 @@ function resolveConversionPlanV1(options) {
|
|
|
158
172
|
const weightOverride = converterConfig?.quantization?.weights ?? null;
|
|
159
173
|
const embedDtypeRaw = normalizeWeightDtype(findTensorDtypeByRole(tensors, 'embedding'));
|
|
160
174
|
const lmHeadDtypeRaw = normalizeWeightDtype(findTensorDtypeByRole(tensors, 'lm_head'));
|
|
161
|
-
const hasVision = hasAnyTensorPattern(tensors, ['vision_', 'vision_tower', 'vision_model', 'image_encoder', 'visual.']);
|
|
175
|
+
const hasVision = hasAnyTensorPattern(tensors, ['vision_', 'vision_tower', 'vision_model', 'image_encoder', 'visual.', 'embed_vision']);
|
|
162
176
|
const hasAudio = hasAnyTensorPattern(tensors, ['audio_', 'audio_encoder', 'whisper', 'wav2vec']);
|
|
163
|
-
const hasProjector = hasAnyTensorPattern(tensors, ['multi_modal_projector', 'mm_projector', 'projector']);
|
|
177
|
+
const hasProjector = hasAnyTensorPattern(tensors, ['multi_modal_projector', 'mm_projector', 'projector', 'embed_vision']);
|
|
164
178
|
const quantizationInfo = buildQuantizationInfo(
|
|
165
179
|
converterConfig,
|
|
166
180
|
sourceQuantization,
|
|
@@ -173,8 +187,6 @@ function resolveConversionPlanV1(options) {
|
|
|
173
187
|
);
|
|
174
188
|
const manifestQuantization = resolveManifestQuantization(weightOverride, sourceQuantization);
|
|
175
189
|
|
|
176
|
-
const modelType = converterConfig?.modelType ?? rawConfig?.model_type ?? 'transformer';
|
|
177
|
-
|
|
178
190
|
// Warn if tensor count seems low for the declared architecture.
|
|
179
191
|
// A typical transformer layer has at least 4 weight tensors (QKV + O projections),
|
|
180
192
|
// plus embeddings and output head, so numLayers * 4 is a rough lower bound.
|
|
@@ -203,7 +215,7 @@ function resolveConversionPlanV1(options) {
|
|
|
203
215
|
layerPattern: inference.layerPattern ?? { type: 'uniform', globalPattern: null, period: null, offset: null, layerTypes: null },
|
|
204
216
|
chatTemplate: inference.chatTemplate,
|
|
205
217
|
pipeline: inference.pipeline ?? null,
|
|
206
|
-
|
|
218
|
+
session,
|
|
207
219
|
execution,
|
|
208
220
|
};
|
|
209
221
|
|
package/src/converter/core.d.ts
CHANGED
|
@@ -26,6 +26,7 @@ import type {
|
|
|
26
26
|
QuantizationInfoSchema,
|
|
27
27
|
MoEConfigSchema,
|
|
28
28
|
ConversionInfoSchema,
|
|
29
|
+
ConverterManifestConfigSchema,
|
|
29
30
|
} from '../config/schema/index.js';
|
|
30
31
|
import type { ExecutionContractArtifact } from '../config/execution-contract-check.js';
|
|
31
32
|
import type {
|
|
@@ -64,6 +65,42 @@ export type ConvertProgress = ConversionProgressSchema;
|
|
|
64
65
|
/** @deprecated Use ConversionOptionsSchema from config/schema */
|
|
65
66
|
export type ConvertOptions = ConversionOptionsSchema;
|
|
66
67
|
|
|
68
|
+
export interface ConvertTensorTransformInput {
|
|
69
|
+
tensor: {
|
|
70
|
+
name: string;
|
|
71
|
+
shape: number[];
|
|
72
|
+
dtype: string;
|
|
73
|
+
size?: number;
|
|
74
|
+
offset?: number;
|
|
75
|
+
sourcePath?: string | null;
|
|
76
|
+
role?: TensorRole | null;
|
|
77
|
+
};
|
|
78
|
+
tensorData: Uint8Array;
|
|
79
|
+
transformContext?: Record<string, unknown> | null;
|
|
80
|
+
reportProgress?: ((currentBytes: number, totalBytes: number) => void) | null;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export interface ConvertTensorTransformResult {
|
|
84
|
+
tensorData: Uint8Array;
|
|
85
|
+
outDtype?: string | null;
|
|
86
|
+
outLayout?: string | null;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export interface ConvertLargeTensorTransformInput {
|
|
90
|
+
tensor: {
|
|
91
|
+
name: string;
|
|
92
|
+
shape: number[];
|
|
93
|
+
dtype: string;
|
|
94
|
+
size?: number;
|
|
95
|
+
offset?: number;
|
|
96
|
+
sourcePath?: string | null;
|
|
97
|
+
role?: TensorRole | null;
|
|
98
|
+
};
|
|
99
|
+
transformContext?: Record<string, unknown> | null;
|
|
100
|
+
reportProgress?: ((currentBytes: number, totalBytes: number) => void) | null;
|
|
101
|
+
writeChunk: (result: ConvertTensorTransformResult) => Promise<void>;
|
|
102
|
+
}
|
|
103
|
+
|
|
67
104
|
/** @deprecated Use ShardSchema from config/schema */
|
|
68
105
|
export type ShardInfo = ShardSchema;
|
|
69
106
|
|
|
@@ -118,6 +155,9 @@ export interface RDRRManifest {
|
|
|
118
155
|
totalSize: number;
|
|
119
156
|
hashAlgorithm: string;
|
|
120
157
|
eos_token_id: number | number[] | null;
|
|
158
|
+
image_token_id?: number;
|
|
159
|
+
audio_token_id?: number;
|
|
160
|
+
video_token_id?: number;
|
|
121
161
|
conversion?: ConversionInfoSchema;
|
|
122
162
|
tokenizer?: TokenizerInfo;
|
|
123
163
|
metadata: {
|
|
@@ -139,6 +179,7 @@ export interface CreateManifestOptions {
|
|
|
139
179
|
eosTokenId?: number | number[] | null;
|
|
140
180
|
convertedAt?: string | null;
|
|
141
181
|
conversionInfo?: ConversionInfoSchema | null;
|
|
182
|
+
manifestConfig?: ConverterManifestConfigSchema | null;
|
|
142
183
|
}
|
|
143
184
|
|
|
144
185
|
/**
|
|
@@ -259,5 +300,11 @@ export declare function createManifest(
|
|
|
259
300
|
export declare function convertModel(
|
|
260
301
|
model: ParsedModel,
|
|
261
302
|
io: ConvertIO,
|
|
262
|
-
options?: ConvertOptions
|
|
303
|
+
options?: ConvertOptions & {
|
|
304
|
+
tensorTransformer?: ((input: ConvertTensorTransformInput) => Promise<ConvertTensorTransformResult> | ConvertTensorTransformResult) | null;
|
|
305
|
+
largeTensorTransformer?: ((input: ConvertLargeTensorTransformInput) => Promise<{
|
|
306
|
+
outDtype?: string | null;
|
|
307
|
+
outLayout?: string | null;
|
|
308
|
+
} | void>) | null;
|
|
309
|
+
}
|
|
263
310
|
): Promise<ConvertResult>;
|
package/src/converter/core.js
CHANGED
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
import { buildExecutionContractArtifact } from '../config/execution-contract-check.js';
|
|
27
27
|
import { buildManifestRequiredInferenceFieldsArtifact } from '../config/required-inference-fields-contract-check.js';
|
|
28
28
|
import { resolveEosTokenId } from './tokenizer-utils.js';
|
|
29
|
+
import { inferBundledTokenizerBehaviorFlags } from '../inference/tokenizers/behavior-flags.js';
|
|
29
30
|
import {
|
|
30
31
|
normalizeQ4KLayout,
|
|
31
32
|
resolveManifestQuantization,
|
|
@@ -210,9 +211,13 @@ function normalizeTensorName(tensor) {
|
|
|
210
211
|
function shouldExcludeTextOnlyTensor(name) {
|
|
211
212
|
const lower = name.toLowerCase();
|
|
212
213
|
return lower.startsWith('vision_tower.')
|
|
214
|
+
|| lower.startsWith('model.vision_tower.')
|
|
213
215
|
|| lower.startsWith('vision_model.')
|
|
216
|
+
|| lower.startsWith('model.vision_model.')
|
|
214
217
|
|| lower.startsWith('visual.')
|
|
215
218
|
|| lower.startsWith('model.visual.')
|
|
219
|
+
|| lower.startsWith('embed_vision.')
|
|
220
|
+
|| lower.startsWith('model.embed_vision.')
|
|
216
221
|
|| lower.startsWith('vision.')
|
|
217
222
|
|| lower.startsWith('model.vision.')
|
|
218
223
|
|| lower.startsWith('vision_encoder.')
|
|
@@ -221,7 +226,9 @@ function shouldExcludeTextOnlyTensor(name) {
|
|
|
221
226
|
|| lower.startsWith('image.')
|
|
222
227
|
|| lower.startsWith('model.image.')
|
|
223
228
|
|| lower.startsWith('audio_tower.')
|
|
229
|
+
|| lower.startsWith('model.audio_tower.')
|
|
224
230
|
|| lower.startsWith('audio_model.')
|
|
231
|
+
|| lower.startsWith('model.audio_model.')
|
|
225
232
|
|| lower.startsWith('audio.')
|
|
226
233
|
|| lower.startsWith('model.audio.')
|
|
227
234
|
|| lower.startsWith('audio_encoder.')
|
|
@@ -475,7 +482,7 @@ function normalizeMoEConfig(config, contextLabel) {
|
|
|
475
482
|
};
|
|
476
483
|
}
|
|
477
484
|
|
|
478
|
-
function resolveManifestMoEConfig(model, options, rawConfig, resolvedModelType) {
|
|
485
|
+
export function resolveManifestMoEConfig(model, options, rawConfig, resolvedModelType) {
|
|
479
486
|
const explicit = normalizeMoEConfig(options?.moeConfig ?? null, options?.modelId ?? 'model');
|
|
480
487
|
if (explicit) return explicit;
|
|
481
488
|
|
|
@@ -496,6 +503,7 @@ function resolveManifestMoEConfig(model, options, rawConfig, resolvedModelType)
|
|
|
496
503
|
|
|
497
504
|
const numExpertsPerToken = resolveMoEConfigNumber(
|
|
498
505
|
rawConfig,
|
|
506
|
+
'top_k_experts',
|
|
499
507
|
'num_experts_per_tok',
|
|
500
508
|
'num_experts_per_token',
|
|
501
509
|
'experts_per_token',
|
|
@@ -505,7 +513,7 @@ function resolveManifestMoEConfig(model, options, rawConfig, resolvedModelType)
|
|
|
505
513
|
if (!numExpertsPerToken) {
|
|
506
514
|
throw new Error(
|
|
507
515
|
`MoE model "${options?.modelId ?? 'model'}" missing experts-per-token config ` +
|
|
508
|
-
'(expected num_experts_per_tok/num_experts_per_token/experts_per_token)'
|
|
516
|
+
'(expected top_k_experts/num_experts_per_tok/num_experts_per_token/experts_per_token)'
|
|
509
517
|
);
|
|
510
518
|
}
|
|
511
519
|
|
|
@@ -522,7 +530,7 @@ function resolveManifestMoEConfig(model, options, rawConfig, resolvedModelType)
|
|
|
522
530
|
);
|
|
523
531
|
}
|
|
524
532
|
|
|
525
|
-
function buildSentencepieceTokenizer(tokenizerConfig, rawConfig, architecture, modelTokenizerModel) {
|
|
533
|
+
export function buildSentencepieceTokenizer(tokenizerConfig, rawConfig, architecture, modelTokenizerModel) {
|
|
526
534
|
if (!modelTokenizerModel) return null;
|
|
527
535
|
|
|
528
536
|
const vocabSize = resolveTokenizerVocabSize(tokenizerConfig, rawConfig, architecture);
|
|
@@ -581,7 +589,7 @@ export function resolveBundledTokenizerVocabSize(tokenizerJson) {
|
|
|
581
589
|
return 0;
|
|
582
590
|
}
|
|
583
591
|
|
|
584
|
-
function buildBundledTokenizer(tokenizerJson, tokenizerConfig, rawConfig) {
|
|
592
|
+
export function buildBundledTokenizer(tokenizerJson, tokenizerConfig, rawConfig) {
|
|
585
593
|
const vocabSize = resolveBundledTokenizerVocabSize(tokenizerJson);
|
|
586
594
|
if (!vocabSize) {
|
|
587
595
|
throw new Error('Tokenizer vocab is missing or empty');
|
|
@@ -602,9 +610,12 @@ function buildBundledTokenizer(tokenizerJson, tokenizerConfig, rawConfig) {
|
|
|
602
610
|
?? resolveTokenizerField(tokenizerConfig, 'add_eos_token', 'addEosToken')
|
|
603
611
|
?? resolveConfigBoolean(rawConfig, 'add_eos_token', 'addEosToken')
|
|
604
612
|
);
|
|
613
|
+
const inferredFlags = inferBundledTokenizerBehaviorFlags(tokenizerJson);
|
|
605
614
|
|
|
606
615
|
if (addBosToken != null) tokenizer.addBosToken = addBosToken;
|
|
616
|
+
else if (inferredFlags.addBosToken != null) tokenizer.addBosToken = inferredFlags.addBosToken;
|
|
607
617
|
if (addEosToken != null) tokenizer.addEosToken = addEosToken;
|
|
618
|
+
else if (inferredFlags.addEosToken != null) tokenizer.addEosToken = inferredFlags.addEosToken;
|
|
608
619
|
|
|
609
620
|
return tokenizer;
|
|
610
621
|
}
|
|
@@ -967,6 +978,10 @@ export function extractArchitecture(config, ggufConfig) {
|
|
|
967
978
|
const linearKeyHeadDim = fromConfig('linear_key_head_dim');
|
|
968
979
|
const linearValueHeadDim = fromConfig('linear_value_head_dim');
|
|
969
980
|
const linearConvKernelDim = fromConfig('linear_conv_kernel_dim');
|
|
981
|
+
const hiddenSizePerLayerInput = fromConfig('hidden_size_per_layer_input');
|
|
982
|
+
const vocabSizePerLayerInput = fromConfig('vocab_size_per_layer_input');
|
|
983
|
+
const globalHeadDim = fromConfig('global_head_dim');
|
|
984
|
+
const numKvSharedLayers = fromConfig('num_kv_shared_layers');
|
|
970
985
|
const linearNormModeConfigured = normalizeLinearNormMode(
|
|
971
986
|
fromConfigValue('linear_norm_mode'),
|
|
972
987
|
fromConfigValue('linear_norm_shared')
|
|
@@ -1000,6 +1015,10 @@ export function extractArchitecture(config, ggufConfig) {
|
|
|
1000
1015
|
linearKeyHeadDim,
|
|
1001
1016
|
linearValueHeadDim,
|
|
1002
1017
|
linearConvKernelDim,
|
|
1018
|
+
hiddenSizePerLayerInput,
|
|
1019
|
+
vocabSizePerLayerInput,
|
|
1020
|
+
globalHeadDim,
|
|
1021
|
+
numKvSharedLayers,
|
|
1003
1022
|
linearNormMode,
|
|
1004
1023
|
};
|
|
1005
1024
|
}
|
|
@@ -1061,6 +1080,86 @@ function getNestedTextConfig(config) {
|
|
|
1061
1080
|
return null;
|
|
1062
1081
|
}
|
|
1063
1082
|
|
|
1083
|
+
function resolveGemma4TextConfig(rawConfig) {
|
|
1084
|
+
const textConfig = getNestedTextConfig(rawConfig);
|
|
1085
|
+
const modelType = String(textConfig?.model_type ?? rawConfig?.model_type ?? '').trim().toLowerCase();
|
|
1086
|
+
if (modelType !== 'gemma4' && modelType !== 'gemma4_text') {
|
|
1087
|
+
return null;
|
|
1088
|
+
}
|
|
1089
|
+
return textConfig ?? rawConfig ?? null;
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
function collectGemma4UnsupportedTensorFlags(tensors) {
|
|
1093
|
+
const names = Array.isArray(tensors) ? tensors.map((tensor) => normalizeTensorName(tensor).toLowerCase()) : [];
|
|
1094
|
+
const flags = [];
|
|
1095
|
+
if (names.some((name) => name.includes('.experts.gate_up_proj'))) {
|
|
1096
|
+
flags.push('experts.gate_up_proj');
|
|
1097
|
+
}
|
|
1098
|
+
if (names.some((name) => name.includes('.experts.down_proj'))) {
|
|
1099
|
+
flags.push('experts.down_proj');
|
|
1100
|
+
}
|
|
1101
|
+
if (names.some((name) => name.includes('.router.per_expert_scale'))) {
|
|
1102
|
+
flags.push('router.per_expert_scale');
|
|
1103
|
+
}
|
|
1104
|
+
if (names.some((name) => name.includes('.router.scale'))) {
|
|
1105
|
+
flags.push('router.scale');
|
|
1106
|
+
}
|
|
1107
|
+
if (names.some((name) => name.includes('.post_feedforward_layernorm_1.'))) {
|
|
1108
|
+
flags.push('post_feedforward_layernorm_1');
|
|
1109
|
+
}
|
|
1110
|
+
if (names.some((name) => name.includes('.post_feedforward_layernorm_2.'))) {
|
|
1111
|
+
flags.push('post_feedforward_layernorm_2');
|
|
1112
|
+
}
|
|
1113
|
+
if (names.some((name) => name.includes('.pre_feedforward_layernorm_2.'))) {
|
|
1114
|
+
flags.push('pre_feedforward_layernorm_2');
|
|
1115
|
+
}
|
|
1116
|
+
return flags;
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
function assertSupportedGemma4Conversion(model, tensors, modelId) {
|
|
1120
|
+
const rawConfig = model?.config ?? null;
|
|
1121
|
+
const textConfig = resolveGemma4TextConfig(rawConfig);
|
|
1122
|
+
if (!textConfig) return;
|
|
1123
|
+
|
|
1124
|
+
const hiddenSizePerLayerInput = Number(textConfig.hidden_size_per_layer_input ?? 0);
|
|
1125
|
+
if (Number.isFinite(hiddenSizePerLayerInput) && hiddenSizePerLayerInput > 0) {
|
|
1126
|
+
const names = Array.isArray(tensors)
|
|
1127
|
+
? tensors.map((tensor) => String(tensor?.name ?? '').trim())
|
|
1128
|
+
: [];
|
|
1129
|
+
const requiredNames = [
|
|
1130
|
+
'embed_tokens_per_layer.weight',
|
|
1131
|
+
'per_layer_input_gate.weight',
|
|
1132
|
+
'per_layer_projection.weight',
|
|
1133
|
+
'post_per_layer_input_norm.weight',
|
|
1134
|
+
'per_layer_model_projection.weight',
|
|
1135
|
+
'per_layer_projection_norm.weight',
|
|
1136
|
+
];
|
|
1137
|
+
const missing = requiredNames.filter((suffix) => !names.some((name) => name.endsWith(suffix)));
|
|
1138
|
+
if (missing.length > 0) {
|
|
1139
|
+
throw new Error(
|
|
1140
|
+
`Gemma 4 model "${modelId}" declares hidden_size_per_layer_input=${hiddenSizePerLayerInput}, ` +
|
|
1141
|
+
`but the checkpoint is missing required per-layer input tensors: ${missing.join(', ')}.`
|
|
1142
|
+
);
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1146
|
+
if (textConfig.enable_moe_block !== true) {
|
|
1147
|
+
return;
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
const unsupportedFlags = collectGemma4UnsupportedTensorFlags(tensors);
|
|
1151
|
+
if (unsupportedFlags.length === 0 && !modelHasMoETensors({ tensors })) {
|
|
1152
|
+
return;
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
throw new Error(
|
|
1156
|
+
`Gemma 4 model "${modelId}" is not supported yet: Gemma 4 MoE decoder blocks require ` +
|
|
1157
|
+
'Gemma-specific router scaling and dual dense+MoE FFN execution, but current Doppler MoE runtime ' +
|
|
1158
|
+
'only supports Mixtral/GPT-OSS semantics. ' +
|
|
1159
|
+
`Detected: ${unsupportedFlags.length > 0 ? unsupportedFlags.join(', ') : 'Gemma 4 MoE tensors'}.`
|
|
1160
|
+
);
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1064
1163
|
|
|
1065
1164
|
export function buildTensorMap(tensors, shardSize) {
|
|
1066
1165
|
if (!shardSize || shardSize <= 0) {
|
|
@@ -1116,7 +1215,7 @@ export function buildTensorMap(tensors, shardSize) {
|
|
|
1116
1215
|
return tensorMap;
|
|
1117
1216
|
}
|
|
1118
1217
|
|
|
1119
|
-
function resolveConvertedAt(value) {
|
|
1218
|
+
export function resolveConvertedAt(value) {
|
|
1120
1219
|
if (value === undefined || value === null || value === '') {
|
|
1121
1220
|
return new Date().toISOString();
|
|
1122
1221
|
}
|
|
@@ -1130,6 +1229,25 @@ function resolveConvertedAt(value) {
|
|
|
1130
1229
|
return new Date(parsed).toISOString();
|
|
1131
1230
|
}
|
|
1132
1231
|
|
|
1232
|
+
function cloneJsonValue(value) {
|
|
1233
|
+
if (value == null) return value;
|
|
1234
|
+
if (typeof structuredClone === 'function') {
|
|
1235
|
+
return structuredClone(value);
|
|
1236
|
+
}
|
|
1237
|
+
return JSON.parse(JSON.stringify(value));
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
export function resolveManifestMultimodalConfig(rawConfig, manifestConfig = null) {
|
|
1241
|
+
const explicitVisionConfig = manifestConfig?.visionConfig;
|
|
1242
|
+
const explicitAudioConfig = manifestConfig?.audioConfig;
|
|
1243
|
+
const visionConfig = explicitVisionConfig ?? rawConfig?.vision_config ?? null;
|
|
1244
|
+
const audioConfig = explicitAudioConfig ?? rawConfig?.audio_config ?? null;
|
|
1245
|
+
return {
|
|
1246
|
+
vision_config: visionConfig ? cloneJsonValue(visionConfig) : null,
|
|
1247
|
+
audio_config: audioConfig ? cloneJsonValue(audioConfig) : null,
|
|
1248
|
+
};
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1133
1251
|
|
|
1134
1252
|
export function createManifest(
|
|
1135
1253
|
modelId,
|
|
@@ -1160,6 +1278,7 @@ export function createManifest(
|
|
|
1160
1278
|
);
|
|
1161
1279
|
const rawConfig = model.config || {};
|
|
1162
1280
|
const generationConfig = model.generationConfig ?? null;
|
|
1281
|
+
const manifestPolicy = options.manifestConfig ?? null;
|
|
1163
1282
|
const resolvedArchitecture = isDiffusion
|
|
1164
1283
|
? architecture
|
|
1165
1284
|
: resolveIntermediateSizeFromTensors(architecture, model, tensorLocations, rawConfig, modelId);
|
|
@@ -1209,6 +1328,16 @@ export function createManifest(
|
|
|
1209
1328
|
throw new Error('Missing hashAlgorithm for manifest');
|
|
1210
1329
|
}
|
|
1211
1330
|
|
|
1331
|
+
const multimodalConfig = isDiffusion
|
|
1332
|
+
? { vision_config: null, audio_config: null }
|
|
1333
|
+
: resolveManifestMultimodalConfig(rawConfig, manifestPolicy);
|
|
1334
|
+
const manifestConfig = isDiffusion
|
|
1335
|
+
? rawConfig
|
|
1336
|
+
: {
|
|
1337
|
+
...(multimodalConfig.vision_config ? { vision_config: multimodalConfig.vision_config } : {}),
|
|
1338
|
+
...(multimodalConfig.audio_config ? { audio_config: multimodalConfig.audio_config } : {}),
|
|
1339
|
+
};
|
|
1340
|
+
|
|
1212
1341
|
const manifest = {
|
|
1213
1342
|
version: RDRR_VERSION,
|
|
1214
1343
|
modelId,
|
|
@@ -1223,7 +1352,10 @@ export function createManifest(
|
|
|
1223
1352
|
totalSize: shards.reduce((sum, s) => sum + s.size, 0),
|
|
1224
1353
|
hashAlgorithm,
|
|
1225
1354
|
eos_token_id: eosTokenId,
|
|
1226
|
-
|
|
1355
|
+
...(rawConfig.image_token_id !== undefined ? { image_token_id: rawConfig.image_token_id } : {}),
|
|
1356
|
+
...(rawConfig.audio_token_id !== undefined ? { audio_token_id: rawConfig.audio_token_id } : {}),
|
|
1357
|
+
...(rawConfig.video_token_id !== undefined ? { video_token_id: rawConfig.video_token_id } : {}),
|
|
1358
|
+
config: Object.keys(manifestConfig).length > 0 ? manifestConfig : undefined,
|
|
1227
1359
|
conversion: options.conversionInfo,
|
|
1228
1360
|
metadata: {
|
|
1229
1361
|
source,
|
|
@@ -1262,6 +1394,7 @@ export function createManifest(
|
|
|
1262
1394
|
// Main Converter (uses I/O adapter)
|
|
1263
1395
|
// ============================================================================
|
|
1264
1396
|
|
|
1397
|
+
const MAX_TENSOR_TYPED_ARRAY_BYTES = 0xffff_ffff;
|
|
1265
1398
|
|
|
1266
1399
|
export async function convertModel(model, io, options = {}) {
|
|
1267
1400
|
const { onProgress, signal } = options;
|
|
@@ -1291,6 +1424,7 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1291
1424
|
}
|
|
1292
1425
|
throw new Error('Missing tensors for conversion');
|
|
1293
1426
|
}
|
|
1427
|
+
assertSupportedGemma4Conversion(model, tensors, modelId);
|
|
1294
1428
|
const totalTensors = tensors.length;
|
|
1295
1429
|
const targetQuant = String(options.quantization ?? model.quantization ?? '').trim().toLowerCase();
|
|
1296
1430
|
const tensorGroupModelType = String(options.modelType ?? model.modelType ?? 'transformer');
|
|
@@ -1331,6 +1465,37 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1331
1465
|
currentShardSize = 0;
|
|
1332
1466
|
};
|
|
1333
1467
|
|
|
1468
|
+
const appendTensorBytes = async (tensorData, tensorSpans) => {
|
|
1469
|
+
if (!(tensorData instanceof Uint8Array)) {
|
|
1470
|
+
throw new Error('appendTensorBytes requires Uint8Array data.');
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
let remainingOffset = 0;
|
|
1474
|
+
while (remainingOffset < tensorData.length) {
|
|
1475
|
+
const availableInShard = shardSize - currentShardSize;
|
|
1476
|
+
const remainingSize = tensorData.length - remainingOffset;
|
|
1477
|
+
const chunkSize = Math.min(remainingSize, availableInShard);
|
|
1478
|
+
const chunk = tensorData.subarray(remainingOffset, remainingOffset + chunkSize);
|
|
1479
|
+
currentShardBuffer.set(chunk, currentShardSize);
|
|
1480
|
+
|
|
1481
|
+
const chunkOffset = currentShardSize;
|
|
1482
|
+
currentShardSize += chunkSize;
|
|
1483
|
+
totalSize += chunkSize;
|
|
1484
|
+
|
|
1485
|
+
tensorSpans.push({
|
|
1486
|
+
shardIndex: currentShardIndex,
|
|
1487
|
+
offset: chunkOffset,
|
|
1488
|
+
size: chunkSize,
|
|
1489
|
+
});
|
|
1490
|
+
|
|
1491
|
+
remainingOffset += chunkSize;
|
|
1492
|
+
|
|
1493
|
+
if (currentShardSize >= shardSize) {
|
|
1494
|
+
await flushShard();
|
|
1495
|
+
}
|
|
1496
|
+
}
|
|
1497
|
+
};
|
|
1498
|
+
|
|
1334
1499
|
// Process tensors
|
|
1335
1500
|
for (let i = 0; i < tensors.length; i++) {
|
|
1336
1501
|
if (signal?.aborted) {
|
|
@@ -1347,9 +1512,6 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1347
1512
|
percent: Math.round(((i + 1) / totalTensors) * 100),
|
|
1348
1513
|
});
|
|
1349
1514
|
|
|
1350
|
-
// Read tensor data
|
|
1351
|
-
const data = await io.readTensorData(tensor);
|
|
1352
|
-
const tensorDataInput = new Uint8Array(data);
|
|
1353
1515
|
const transformContext = {
|
|
1354
1516
|
targetQuant,
|
|
1355
1517
|
q4kLayout,
|
|
@@ -1357,66 +1519,90 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1357
1519
|
quantizeEmbeddings,
|
|
1358
1520
|
modulesToNotConvert,
|
|
1359
1521
|
};
|
|
1360
|
-
const
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
percent: Math.round(((i + 1) / totalTensors) * 100),
|
|
1374
|
-
tensorName: tensor.name,
|
|
1375
|
-
tensorBytesCurrent: currentBytes,
|
|
1376
|
-
tensorBytesTotal: totalBytes,
|
|
1377
|
-
});
|
|
1378
|
-
},
|
|
1379
|
-
})
|
|
1380
|
-
: transformTensorBytes(tensor, tensorDataInput, transformContext)
|
|
1381
|
-
);
|
|
1382
|
-
|
|
1383
|
-
let tensorData = transformResult?.tensorData;
|
|
1384
|
-
if (!(tensorData instanceof Uint8Array)) {
|
|
1385
|
-
throw new Error(`Tensor transformer must return Uint8Array data for ${tensor.name}.`);
|
|
1386
|
-
}
|
|
1387
|
-
const outDtype = transformResult?.outDtype ?? tensor.dtype;
|
|
1388
|
-
const outLayout = transformResult?.outLayout ?? null;
|
|
1389
|
-
|
|
1390
|
-
const tensorSize = tensorData.byteLength;
|
|
1391
|
-
|
|
1392
|
-
// Track tensor location
|
|
1522
|
+
const reportTensorProgress = (currentBytes, totalBytes) => {
|
|
1523
|
+
if (!Number.isFinite(currentBytes) || !Number.isFinite(totalBytes)) return;
|
|
1524
|
+
onProgress?.({
|
|
1525
|
+
stage: ConvertStage.WRITING,
|
|
1526
|
+
message: `Processing ${tensor.name}`,
|
|
1527
|
+
current: i + 1,
|
|
1528
|
+
total: totalTensors,
|
|
1529
|
+
percent: Math.round(((i + 1) / totalTensors) * 100),
|
|
1530
|
+
tensorName: tensor.name,
|
|
1531
|
+
tensorBytesCurrent: currentBytes,
|
|
1532
|
+
tensorBytesTotal: totalBytes,
|
|
1533
|
+
});
|
|
1534
|
+
};
|
|
1393
1535
|
const tensorSpans = [];
|
|
1536
|
+
const sourceTensorSize = Number.isFinite(tensor?.size) ? Number(tensor.size) : null;
|
|
1537
|
+
let outDtype = tensor.dtype;
|
|
1538
|
+
let outLayout = null;
|
|
1539
|
+
let tensorSize = 0;
|
|
1394
1540
|
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
currentShardSize += chunkSize;
|
|
1406
|
-
totalSize += chunkSize;
|
|
1541
|
+
if (
|
|
1542
|
+
sourceTensorSize != null
|
|
1543
|
+
&& sourceTensorSize > MAX_TENSOR_TYPED_ARRAY_BYTES
|
|
1544
|
+
) {
|
|
1545
|
+
if (typeof options.largeTensorTransformer !== 'function') {
|
|
1546
|
+
throw new Error(
|
|
1547
|
+
`Tensor "${tensor.name}" is ${formatBytes(sourceTensorSize)} and exceeds the single-buffer conversion limit ` +
|
|
1548
|
+
`(${formatBytes(MAX_TENSOR_TYPED_ARRAY_BYTES)}). Provide a largeTensorTransformer for streamed conversion.`
|
|
1549
|
+
);
|
|
1550
|
+
}
|
|
1407
1551
|
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1552
|
+
let emittedChunk = false;
|
|
1553
|
+
await options.largeTensorTransformer({
|
|
1554
|
+
tensor,
|
|
1555
|
+
transformContext,
|
|
1556
|
+
reportProgress: reportTensorProgress,
|
|
1557
|
+
async writeChunk(result) {
|
|
1558
|
+
const tensorData = result?.tensorData;
|
|
1559
|
+
if (!(tensorData instanceof Uint8Array)) {
|
|
1560
|
+
throw new Error(`Large tensor transformer must return Uint8Array data for ${tensor.name}.`);
|
|
1561
|
+
}
|
|
1562
|
+
const chunkOutDtype = result?.outDtype ?? tensor.dtype;
|
|
1563
|
+
const chunkOutLayout = result?.outLayout ?? null;
|
|
1564
|
+
if (!emittedChunk) {
|
|
1565
|
+
outDtype = chunkOutDtype;
|
|
1566
|
+
outLayout = chunkOutLayout;
|
|
1567
|
+
emittedChunk = true;
|
|
1568
|
+
} else {
|
|
1569
|
+
if (chunkOutDtype !== outDtype) {
|
|
1570
|
+
throw new Error(`Large tensor transformer returned inconsistent dtype for ${tensor.name}.`);
|
|
1571
|
+
}
|
|
1572
|
+
if (chunkOutLayout !== outLayout) {
|
|
1573
|
+
throw new Error(`Large tensor transformer returned inconsistent layout for ${tensor.name}.`);
|
|
1574
|
+
}
|
|
1575
|
+
}
|
|
1576
|
+
tensorSize += tensorData.byteLength;
|
|
1577
|
+
await appendTensorBytes(tensorData, tensorSpans);
|
|
1578
|
+
},
|
|
1412
1579
|
});
|
|
1413
1580
|
|
|
1414
|
-
|
|
1581
|
+
if (!emittedChunk) {
|
|
1582
|
+
throw new Error(`Large tensor transformer did not emit any bytes for ${tensor.name}.`);
|
|
1583
|
+
}
|
|
1584
|
+
} else {
|
|
1585
|
+
const data = await io.readTensorData(tensor);
|
|
1586
|
+
const tensorDataInput = new Uint8Array(data);
|
|
1587
|
+
const transformResult = (
|
|
1588
|
+
typeof options.tensorTransformer === 'function'
|
|
1589
|
+
? await options.tensorTransformer({
|
|
1590
|
+
tensor,
|
|
1591
|
+
tensorData: tensorDataInput,
|
|
1592
|
+
transformContext,
|
|
1593
|
+
reportProgress: reportTensorProgress,
|
|
1594
|
+
})
|
|
1595
|
+
: transformTensorBytes(tensor, tensorDataInput, transformContext)
|
|
1596
|
+
);
|
|
1415
1597
|
|
|
1416
|
-
|
|
1417
|
-
if (
|
|
1418
|
-
|
|
1598
|
+
const tensorData = transformResult?.tensorData;
|
|
1599
|
+
if (!(tensorData instanceof Uint8Array)) {
|
|
1600
|
+
throw new Error(`Tensor transformer must return Uint8Array data for ${tensor.name}.`);
|
|
1419
1601
|
}
|
|
1602
|
+
outDtype = transformResult?.outDtype ?? tensor.dtype;
|
|
1603
|
+
outLayout = transformResult?.outLayout ?? null;
|
|
1604
|
+
tensorSize = tensorData.byteLength;
|
|
1605
|
+
await appendTensorBytes(tensorData, tensorSpans);
|
|
1420
1606
|
}
|
|
1421
1607
|
|
|
1422
1608
|
// Record tensor location
|
|
@@ -1487,6 +1673,7 @@ export async function convertModel(model, io, options = {}) {
|
|
|
1487
1673
|
eosTokenId: options.eosTokenId,
|
|
1488
1674
|
convertedAt: converterConfig?.manifest?.conversion?.convertedAt ?? null,
|
|
1489
1675
|
conversionInfo: converterConfig?.manifest?.conversion ?? null,
|
|
1676
|
+
manifestConfig: converterConfig?.manifest ?? null,
|
|
1490
1677
|
});
|
|
1491
1678
|
|
|
1492
1679
|
// Write manifest
|
|
@@ -1,4 +1,13 @@
|
|
|
1
|
-
|
|
1
|
+
export interface TensorSource {
|
|
2
|
+
sourceType?: string;
|
|
3
|
+
name?: string | null;
|
|
4
|
+
size: number;
|
|
5
|
+
file?: Blob | File;
|
|
6
|
+
readRange: (offset: number, length: number) => Promise<ArrayBuffer>;
|
|
7
|
+
readAll?: () => Promise<ArrayBuffer>;
|
|
8
|
+
close?: () => Promise<void>;
|
|
9
|
+
getAuxFiles?: () => Promise<Record<string, unknown>> | Record<string, unknown>;
|
|
10
|
+
}
|
|
2
11
|
|
|
3
12
|
export interface GGUFParserAdapter {
|
|
4
13
|
file: File | TensorSource;
|
|
@@ -8,6 +8,7 @@ import type { ManifestInferenceSchema } from '../config/schema/manifest.schema.j
|
|
|
8
8
|
* If absent, Gemma 3 style rope_parameters are mapped from:
|
|
9
9
|
* - full_attention: ropeTheta + global scaling
|
|
10
10
|
* - sliding_attention: ropeLocalTheta + local scaling
|
|
11
|
+
* - proportional RoPE: explicit frequency-base dims for non-uniform head geometry
|
|
11
12
|
*/
|
|
12
13
|
export declare function buildRoPEConfig(
|
|
13
14
|
converterInference: InferenceConfigSchema,
|