doppler-gpu 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
import { log } from '../../../debug/index.js';
|
|
2
2
|
import { mergeConfig, dumpConfigSources } from '../../../config/merge.js';
|
|
3
3
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
4
|
+
import {
|
|
5
|
+
PER_LAYER_INPUT_MATERIALIZATION_MODES,
|
|
6
|
+
PER_LAYER_INPUT_ROW_CACHE_MODES,
|
|
7
|
+
PER_LAYER_INPUT_PREFETCH_MODES,
|
|
8
|
+
PER_LAYER_INPUT_GPU_UPLOAD_MODES,
|
|
9
|
+
PER_LAYER_INPUT_HOT_CACHE_MODES,
|
|
10
|
+
} from '../../../config/schema/execution-v1.schema.js';
|
|
4
11
|
|
|
5
12
|
const UNSUPPORTED_RUNTIME_MODEL_TYPES = new Set(['mamba', 'rwkv']);
|
|
6
13
|
|
|
@@ -11,6 +18,7 @@ const UNSUPPORTED_RUNTIME_MODEL_TYPES = new Set(['mamba', 'rwkv']);
|
|
|
11
18
|
*/
|
|
12
19
|
const KNOWN_CHAT_TEMPLATE_TYPES = new Set([
|
|
13
20
|
'gemma',
|
|
21
|
+
'gemma4',
|
|
14
22
|
'llama3',
|
|
15
23
|
'gpt-oss',
|
|
16
24
|
'chatml',
|
|
@@ -20,8 +28,7 @@ const KNOWN_CHAT_TEMPLATE_TYPES = new Set([
|
|
|
20
28
|
|
|
21
29
|
/**
|
|
22
30
|
* Validate that a chatTemplate.type value is either null (disabled) or a known
|
|
23
|
-
* formatter type.
|
|
24
|
-
* without a breaking throw.
|
|
31
|
+
* formatter type.
|
|
25
32
|
*
|
|
26
33
|
* @param {string | null} type - The chatTemplate.type from manifest inference.
|
|
27
34
|
* @param {string} modelId - Model identifier for diagnostic messages.
|
|
@@ -30,13 +37,163 @@ const KNOWN_CHAT_TEMPLATE_TYPES = new Set([
|
|
|
30
37
|
export function validateChatTemplateType(type, modelId) {
|
|
31
38
|
if (type === null || type === undefined) return true;
|
|
32
39
|
if (KNOWN_CHAT_TEMPLATE_TYPES.has(type)) return true;
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
`
|
|
36
|
-
`formatter type. Known types: ${[...KNOWN_CHAT_TEMPLATE_TYPES].join(', ')}. ` +
|
|
37
|
-
`The formatter will fall back to plaintext at generation time.`
|
|
40
|
+
throw new Error(
|
|
41
|
+
`Manifest "${modelId}" declares chatTemplate.type="${type}" which is not a known formatter type. ` +
|
|
42
|
+
`Known types: ${[...KNOWN_CHAT_TEMPLATE_TYPES].join(', ')}. Re-convert the model or fix the manifest.`
|
|
38
43
|
);
|
|
39
|
-
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function resolvePerLayerInputsSession(inferenceConfig, modelId) {
|
|
47
|
+
const sessionConfig = inferenceConfig?.session?.perLayerInputs;
|
|
48
|
+
if (sessionConfig === undefined) {
|
|
49
|
+
throw new Error(
|
|
50
|
+
`Manifest "${modelId}" is missing inference.session.perLayerInputs. ` +
|
|
51
|
+
'Re-convert the model so per-layer input materialization policy is explicit.'
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
if (!sessionConfig || typeof sessionConfig !== 'object' || Array.isArray(sessionConfig)) {
|
|
55
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.perLayerInputs.`);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const materialization = sessionConfig.materialization;
|
|
59
|
+
if (!PER_LAYER_INPUT_MATERIALIZATION_MODES.includes(materialization)) {
|
|
60
|
+
throw new Error(
|
|
61
|
+
`Manifest "${modelId}" has invalid inference.session.perLayerInputs.materialization ` +
|
|
62
|
+
`"${String(materialization)}".`
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const rowCache = sessionConfig.rowCache;
|
|
67
|
+
if (!rowCache || typeof rowCache !== 'object' || Array.isArray(rowCache)) {
|
|
68
|
+
throw new Error(`Manifest "${modelId}" is missing inference.session.perLayerInputs.rowCache.`);
|
|
69
|
+
}
|
|
70
|
+
if (!PER_LAYER_INPUT_ROW_CACHE_MODES.includes(rowCache.mode)) {
|
|
71
|
+
throw new Error(
|
|
72
|
+
`Manifest "${modelId}" has invalid inference.session.perLayerInputs.rowCache.mode ` +
|
|
73
|
+
`"${String(rowCache.mode)}".`
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const maxRows = Math.trunc(Number(rowCache.maxRows));
|
|
78
|
+
if (!Number.isFinite(maxRows) || maxRows <= 0) {
|
|
79
|
+
throw new Error(
|
|
80
|
+
`Manifest "${modelId}" requires inference.session.perLayerInputs.rowCache.maxRows ` +
|
|
81
|
+
`to be a positive integer; got ${String(rowCache.maxRows)}.`
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const maxBytes = Math.trunc(Number(rowCache.maxBytes));
|
|
86
|
+
if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
|
|
87
|
+
throw new Error(
|
|
88
|
+
`Manifest "${modelId}" requires inference.session.perLayerInputs.rowCache.maxBytes ` +
|
|
89
|
+
`to be a positive integer; got ${String(rowCache.maxBytes)}.`
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const decodedDtype = String(rowCache.decodedDtype ?? '').toLowerCase();
|
|
94
|
+
if (decodedDtype !== 'f16' && decodedDtype !== 'f32') {
|
|
95
|
+
throw new Error(
|
|
96
|
+
`Manifest "${modelId}" has invalid inference.session.perLayerInputs.rowCache.decodedDtype ` +
|
|
97
|
+
`"${String(rowCache.decodedDtype)}".`
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const prefetch = sessionConfig.prefetch;
|
|
102
|
+
if (!prefetch || typeof prefetch !== 'object' || Array.isArray(prefetch)) {
|
|
103
|
+
throw new Error(`Manifest "${modelId}" is missing inference.session.perLayerInputs.prefetch.`);
|
|
104
|
+
}
|
|
105
|
+
if (!PER_LAYER_INPUT_PREFETCH_MODES.includes(prefetch.mode)) {
|
|
106
|
+
throw new Error(
|
|
107
|
+
`Manifest "${modelId}" has invalid inference.session.perLayerInputs.prefetch.mode ` +
|
|
108
|
+
`"${String(prefetch.mode)}".`
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const rowsAhead = Math.trunc(Number(prefetch.rowsAhead));
|
|
113
|
+
if (!Number.isFinite(rowsAhead) || rowsAhead <= 0) {
|
|
114
|
+
throw new Error(
|
|
115
|
+
`Manifest "${modelId}" requires inference.session.perLayerInputs.prefetch.rowsAhead ` +
|
|
116
|
+
`to be a positive integer; got ${String(prefetch.rowsAhead)}.`
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const gpuUpload = sessionConfig.gpuUpload;
|
|
121
|
+
if (!gpuUpload || typeof gpuUpload !== 'object' || Array.isArray(gpuUpload)) {
|
|
122
|
+
throw new Error(`Manifest "${modelId}" is missing inference.session.perLayerInputs.gpuUpload.`);
|
|
123
|
+
}
|
|
124
|
+
if (!PER_LAYER_INPUT_GPU_UPLOAD_MODES.includes(gpuUpload.mode)) {
|
|
125
|
+
throw new Error(
|
|
126
|
+
`Manifest "${modelId}" has invalid inference.session.perLayerInputs.gpuUpload.mode ` +
|
|
127
|
+
`"${String(gpuUpload.mode)}".`
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const stagingRows = Math.trunc(Number(gpuUpload.stagingRows));
|
|
132
|
+
if (!Number.isFinite(stagingRows) || stagingRows <= 0) {
|
|
133
|
+
throw new Error(
|
|
134
|
+
`Manifest "${modelId}" requires inference.session.perLayerInputs.gpuUpload.stagingRows ` +
|
|
135
|
+
`to be a positive integer; got ${String(gpuUpload.stagingRows)}.`
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const hotCache = sessionConfig.hotCache;
|
|
140
|
+
if (!hotCache || typeof hotCache !== 'object' || Array.isArray(hotCache)) {
|
|
141
|
+
throw new Error(`Manifest "${modelId}" is missing inference.session.perLayerInputs.hotCache.`);
|
|
142
|
+
}
|
|
143
|
+
if (!PER_LAYER_INPUT_HOT_CACHE_MODES.includes(hotCache.mode)) {
|
|
144
|
+
throw new Error(
|
|
145
|
+
`Manifest "${modelId}" has invalid inference.session.perLayerInputs.hotCache.mode ` +
|
|
146
|
+
`"${String(hotCache.mode)}".`
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const hotMaxTokens = Math.trunc(Number(hotCache.maxTokens));
|
|
151
|
+
if (!Number.isFinite(hotMaxTokens) || hotMaxTokens <= 0) {
|
|
152
|
+
throw new Error(
|
|
153
|
+
`Manifest "${modelId}" requires inference.session.perLayerInputs.hotCache.maxTokens ` +
|
|
154
|
+
`to be a positive integer; got ${String(hotCache.maxTokens)}.`
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const hotMaxBytes = Math.trunc(Number(hotCache.maxBytes));
|
|
159
|
+
if (!Number.isFinite(hotMaxBytes) || hotMaxBytes <= 0) {
|
|
160
|
+
throw new Error(
|
|
161
|
+
`Manifest "${modelId}" requires inference.session.perLayerInputs.hotCache.maxBytes ` +
|
|
162
|
+
`to be a positive integer; got ${String(hotCache.maxBytes)}.`
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const hotOutputDtype = String(hotCache.outputDtype ?? '').toLowerCase();
|
|
167
|
+
if (hotOutputDtype !== 'f16' && hotOutputDtype !== 'f32') {
|
|
168
|
+
throw new Error(
|
|
169
|
+
`Manifest "${modelId}" has invalid inference.session.perLayerInputs.hotCache.outputDtype ` +
|
|
170
|
+
`"${String(hotCache.outputDtype)}".`
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
return {
|
|
175
|
+
materialization,
|
|
176
|
+
rowCache: {
|
|
177
|
+
mode: rowCache.mode,
|
|
178
|
+
maxRows,
|
|
179
|
+
maxBytes,
|
|
180
|
+
decodedDtype,
|
|
181
|
+
},
|
|
182
|
+
prefetch: {
|
|
183
|
+
mode: prefetch.mode,
|
|
184
|
+
rowsAhead,
|
|
185
|
+
},
|
|
186
|
+
gpuUpload: {
|
|
187
|
+
mode: gpuUpload.mode,
|
|
188
|
+
stagingRows,
|
|
189
|
+
},
|
|
190
|
+
hotCache: {
|
|
191
|
+
mode: hotCache.mode,
|
|
192
|
+
maxTokens: hotMaxTokens,
|
|
193
|
+
maxBytes: hotMaxBytes,
|
|
194
|
+
outputDtype: hotOutputDtype,
|
|
195
|
+
},
|
|
196
|
+
};
|
|
40
197
|
}
|
|
41
198
|
|
|
42
199
|
// =============================================================================
|
|
@@ -78,6 +235,32 @@ function resolveRotaryDim(headDim, partialRotaryFactor, modelId) {
|
|
|
78
235
|
return rotaryDim;
|
|
79
236
|
}
|
|
80
237
|
|
|
238
|
+
function resolveFrequencyBaseDim(headDim, rotaryDim, frequencyBaseDim, modelId, fieldName) {
|
|
239
|
+
if (frequencyBaseDim == null) {
|
|
240
|
+
return rotaryDim;
|
|
241
|
+
}
|
|
242
|
+
if (typeof frequencyBaseDim !== 'number' || Number.isNaN(frequencyBaseDim)) {
|
|
243
|
+
throw new Error(`Manifest "${modelId}" has invalid ${fieldName}.`);
|
|
244
|
+
}
|
|
245
|
+
const resolved = Math.trunc(frequencyBaseDim);
|
|
246
|
+
if (resolved <= 0 || (resolved % 2) !== 0) {
|
|
247
|
+
throw new Error(
|
|
248
|
+
`Manifest "${modelId}" requires ${fieldName} to be a positive even integer; got ${frequencyBaseDim}.`
|
|
249
|
+
);
|
|
250
|
+
}
|
|
251
|
+
if (resolved < rotaryDim) {
|
|
252
|
+
throw new Error(
|
|
253
|
+
`Manifest "${modelId}" requires ${fieldName} (${resolved}) to be >= rotary dim (${rotaryDim}).`
|
|
254
|
+
);
|
|
255
|
+
}
|
|
256
|
+
if (resolved > headDim) {
|
|
257
|
+
throw new Error(
|
|
258
|
+
`Manifest "${modelId}" requires ${fieldName} (${resolved}) to be <= attention head dim (${headDim}).`
|
|
259
|
+
);
|
|
260
|
+
}
|
|
261
|
+
return resolved;
|
|
262
|
+
}
|
|
263
|
+
|
|
81
264
|
export function getStopTokenIds(manifest) {
|
|
82
265
|
const eosTokenId = manifest?.eos_token_id;
|
|
83
266
|
if (Array.isArray(eosTokenId)) return eosTokenId;
|
|
@@ -189,6 +372,170 @@ function resolveIntermediateSizeForRuntime(manifest, inf, arch, modelId) {
|
|
|
189
372
|
);
|
|
190
373
|
}
|
|
191
374
|
|
|
375
|
+
function buildPerLayerIntermediateSizes({
|
|
376
|
+
baseIntermediateSize,
|
|
377
|
+
numLayers,
|
|
378
|
+
numKvSharedLayers,
|
|
379
|
+
useDoubleWideMlp,
|
|
380
|
+
modelId,
|
|
381
|
+
}) {
|
|
382
|
+
if (!Number.isFinite(baseIntermediateSize) || baseIntermediateSize <= 0) {
|
|
383
|
+
throw new Error(
|
|
384
|
+
`Manifest "${modelId}" has invalid architecture.intermediateSize (${String(baseIntermediateSize)}).`
|
|
385
|
+
);
|
|
386
|
+
}
|
|
387
|
+
if (!Number.isFinite(numLayers) || numLayers <= 0) {
|
|
388
|
+
throw new Error(
|
|
389
|
+
`Manifest "${modelId}" has invalid architecture.numLayers (${String(numLayers)}).`
|
|
390
|
+
);
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
const resolvedBaseIntermediateSize = Math.trunc(baseIntermediateSize);
|
|
394
|
+
const resolvedNumLayers = Math.trunc(numLayers);
|
|
395
|
+
const intermediateSizes = new Array(resolvedNumLayers).fill(resolvedBaseIntermediateSize);
|
|
396
|
+
|
|
397
|
+
if (!useDoubleWideMlp) {
|
|
398
|
+
return intermediateSizes;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
if (!Number.isFinite(numKvSharedLayers) || numKvSharedLayers <= 0 || numKvSharedLayers >= resolvedNumLayers) {
|
|
402
|
+
throw new Error(
|
|
403
|
+
`Manifest "${modelId}" enables ffn.useDoubleWideMlp, but architecture.numKvSharedLayers=${String(numKvSharedLayers)} ` +
|
|
404
|
+
`must be a positive integer smaller than numLayers=${resolvedNumLayers}.`
|
|
405
|
+
);
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
const firstKvSharedLayerIdx = resolvedNumLayers - Math.trunc(numKvSharedLayers);
|
|
409
|
+
if (firstKvSharedLayerIdx <= 0 || firstKvSharedLayerIdx >= resolvedNumLayers) {
|
|
410
|
+
throw new Error(
|
|
411
|
+
`Manifest "${modelId}" enables ffn.useDoubleWideMlp, but the derived first KV-shared layer index ` +
|
|
412
|
+
`(${firstKvSharedLayerIdx}) is invalid for numLayers=${resolvedNumLayers}.`
|
|
413
|
+
);
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
const widenedIntermediateSize = resolvedBaseIntermediateSize * 2;
|
|
417
|
+
for (let layerIdx = firstKvSharedLayerIdx; layerIdx < resolvedNumLayers; layerIdx += 1) {
|
|
418
|
+
intermediateSizes[layerIdx] = widenedIntermediateSize;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
return intermediateSizes;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
function getDenseFfnTensorShape(tensors, names) {
|
|
425
|
+
if (!tensors || typeof tensors !== 'object') return null;
|
|
426
|
+
for (const name of names) {
|
|
427
|
+
const shape = normalizeFfnTensorShape(tensors[name]?.shape);
|
|
428
|
+
if (shape) return shape;
|
|
429
|
+
}
|
|
430
|
+
return null;
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
function assertDenseFfnTensorShape(modelId, layerIdx, label, actualShape, expectedShape) {
|
|
434
|
+
if (!actualShape) return;
|
|
435
|
+
if (actualShape[0] === expectedShape[0] && actualShape[1] === expectedShape[1]) {
|
|
436
|
+
return;
|
|
437
|
+
}
|
|
438
|
+
throw new Error(
|
|
439
|
+
`Manifest "${modelId}" layer ${layerIdx} ${label} shape [${actualShape.join(', ')}] does not match ` +
|
|
440
|
+
`the resolved FFN contract [${expectedShape.join(', ')}]. Re-convert the model so manifest inference ` +
|
|
441
|
+
'and FFN tensor shapes agree.'
|
|
442
|
+
);
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
function validateLayerIntermediateSizesAgainstManifest(manifest, hiddenSize, layerIntermediateSizes, modelId) {
|
|
446
|
+
const tensors = manifest?.tensors;
|
|
447
|
+
if (!tensors || typeof tensors !== 'object' || !Array.isArray(layerIntermediateSizes)) {
|
|
448
|
+
return;
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
for (let layerIdx = 0; layerIdx < layerIntermediateSizes.length; layerIdx += 1) {
|
|
452
|
+
const intermediateSize = Number(layerIntermediateSizes[layerIdx]);
|
|
453
|
+
if (!Number.isFinite(intermediateSize) || intermediateSize <= 0) {
|
|
454
|
+
throw new Error(
|
|
455
|
+
`Manifest "${modelId}" resolved an invalid FFN intermediate size (${String(layerIntermediateSizes[layerIdx])}) ` +
|
|
456
|
+
`for layer ${layerIdx}.`
|
|
457
|
+
);
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
const expectedIntermediateSize = Math.trunc(intermediateSize);
|
|
461
|
+
const languagePrefix = `model.language_model.layers.${layerIdx}`;
|
|
462
|
+
const genericPrefix = `model.layers.${layerIdx}`;
|
|
463
|
+
const gateShape = getDenseFfnTensorShape(tensors, [
|
|
464
|
+
`${languagePrefix}.mlp.gate_proj.weight`,
|
|
465
|
+
`${genericPrefix}.mlp.gate_proj.weight`,
|
|
466
|
+
`${languagePrefix}.ffn.gate_proj.weight`,
|
|
467
|
+
`${languagePrefix}.ffn_gate.weight`,
|
|
468
|
+
`layers.${layerIdx}.feed_forward.w1.weight`,
|
|
469
|
+
]);
|
|
470
|
+
const upShape = getDenseFfnTensorShape(tensors, [
|
|
471
|
+
`${languagePrefix}.mlp.up_proj.weight`,
|
|
472
|
+
`${genericPrefix}.mlp.up_proj.weight`,
|
|
473
|
+
`${languagePrefix}.ffn.up_proj.weight`,
|
|
474
|
+
`${languagePrefix}.ffn_up.weight`,
|
|
475
|
+
`layers.${layerIdx}.feed_forward.w3.weight`,
|
|
476
|
+
]);
|
|
477
|
+
const downShape = getDenseFfnTensorShape(tensors, [
|
|
478
|
+
`${languagePrefix}.mlp.down_proj.weight`,
|
|
479
|
+
`${genericPrefix}.mlp.down_proj.weight`,
|
|
480
|
+
`${languagePrefix}.ffn.down_proj.weight`,
|
|
481
|
+
`${languagePrefix}.ffn_down.weight`,
|
|
482
|
+
`layers.${layerIdx}.feed_forward.w2.weight`,
|
|
483
|
+
]);
|
|
484
|
+
const gateUpShape = getDenseFfnTensorShape(tensors, [
|
|
485
|
+
`${languagePrefix}.mlp.gate_up_proj.weight`,
|
|
486
|
+
`${genericPrefix}.mlp.gate_up_proj.weight`,
|
|
487
|
+
`${languagePrefix}.ffn.gate_up_proj.weight`,
|
|
488
|
+
`${languagePrefix}.ffn_gate_up.weight`,
|
|
489
|
+
`layers.${layerIdx}.feed_forward.w1_w3.weight`,
|
|
490
|
+
]);
|
|
491
|
+
|
|
492
|
+
assertDenseFfnTensorShape(
|
|
493
|
+
modelId,
|
|
494
|
+
layerIdx,
|
|
495
|
+
'gate weight',
|
|
496
|
+
gateShape,
|
|
497
|
+
[expectedIntermediateSize, hiddenSize]
|
|
498
|
+
);
|
|
499
|
+
assertDenseFfnTensorShape(
|
|
500
|
+
modelId,
|
|
501
|
+
layerIdx,
|
|
502
|
+
'up weight',
|
|
503
|
+
upShape,
|
|
504
|
+
[expectedIntermediateSize, hiddenSize]
|
|
505
|
+
);
|
|
506
|
+
assertDenseFfnTensorShape(
|
|
507
|
+
modelId,
|
|
508
|
+
layerIdx,
|
|
509
|
+
'down weight',
|
|
510
|
+
downShape,
|
|
511
|
+
[hiddenSize, expectedIntermediateSize]
|
|
512
|
+
);
|
|
513
|
+
assertDenseFfnTensorShape(
|
|
514
|
+
modelId,
|
|
515
|
+
layerIdx,
|
|
516
|
+
'gate_up weight',
|
|
517
|
+
gateUpShape,
|
|
518
|
+
[expectedIntermediateSize * 2, hiddenSize]
|
|
519
|
+
);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
export function resolveLayerIntermediateSize(config, layerIdx) {
|
|
524
|
+
const intermediateSizes = Array.isArray(config?.intermediateSizes) ? config.intermediateSizes : null;
|
|
525
|
+
if (Number.isFinite(layerIdx) && intermediateSizes) {
|
|
526
|
+
const resolved = intermediateSizes[Math.trunc(layerIdx)];
|
|
527
|
+
if (Number.isFinite(resolved) && resolved > 0) {
|
|
528
|
+
return Math.trunc(resolved);
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
const fallback = Number(config?.intermediateSize);
|
|
533
|
+
if (!Number.isFinite(fallback) || fallback <= 0) {
|
|
534
|
+
throw new Error(`Invalid modelConfig.intermediateSize: ${String(config?.intermediateSize)}`);
|
|
535
|
+
}
|
|
536
|
+
return Math.trunc(fallback);
|
|
537
|
+
}
|
|
538
|
+
|
|
192
539
|
// =============================================================================
|
|
193
540
|
// Manifest-First Config Resolution (NEW)
|
|
194
541
|
// =============================================================================
|
|
@@ -217,6 +564,9 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
217
564
|
if (inf.attention.queryKeyNorm == null) {
|
|
218
565
|
errors.push('attention.queryKeyNorm is required');
|
|
219
566
|
}
|
|
567
|
+
if (inf.attention.valueNorm == null) {
|
|
568
|
+
errors.push('attention.valueNorm is required');
|
|
569
|
+
}
|
|
220
570
|
if (inf.attention.attentionBias == null) {
|
|
221
571
|
errors.push('attention.attentionBias is required');
|
|
222
572
|
}
|
|
@@ -255,6 +605,9 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
255
605
|
if (inf.ffn.gatedActivation == null) {
|
|
256
606
|
errors.push('ffn.gatedActivation is required');
|
|
257
607
|
}
|
|
608
|
+
if (inf.ffn.useDoubleWideMlp == null) {
|
|
609
|
+
errors.push('ffn.useDoubleWideMlp is required');
|
|
610
|
+
}
|
|
258
611
|
if (inf.ffn.swigluLimit === undefined) {
|
|
259
612
|
errors.push('ffn.swigluLimit must be explicitly set (null for no limit, or number)');
|
|
260
613
|
} else {
|
|
@@ -278,6 +631,13 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
278
631
|
if (inf.rope.ropeLocalTheta === undefined) {
|
|
279
632
|
errors.push('rope.ropeLocalTheta must be explicitly set (null for no local theta, or number)');
|
|
280
633
|
}
|
|
634
|
+
if (
|
|
635
|
+
inf.rope.ropeInterleaved !== undefined
|
|
636
|
+
&& inf.rope.ropeInterleaved != null
|
|
637
|
+
&& typeof inf.rope.ropeInterleaved !== 'boolean'
|
|
638
|
+
) {
|
|
639
|
+
errors.push('rope.ropeInterleaved must be boolean when provided');
|
|
640
|
+
}
|
|
281
641
|
if (inf.rope.mropeInterleaved == null) {
|
|
282
642
|
errors.push('rope.mropeInterleaved is required');
|
|
283
643
|
}
|
|
@@ -292,6 +652,30 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
292
652
|
errors.push('rope.partialRotaryFactor must be a number in (0, 1] or null');
|
|
293
653
|
}
|
|
294
654
|
}
|
|
655
|
+
if (inf.rope.ropeLocalPartialRotaryFactor === undefined) {
|
|
656
|
+
errors.push('rope.ropeLocalPartialRotaryFactor must be explicitly set (null when unused, or a number in (0, 1])');
|
|
657
|
+
} else {
|
|
658
|
+
const factor = inf.rope.ropeLocalPartialRotaryFactor;
|
|
659
|
+
if (factor !== null && (typeof factor !== 'number' || Number.isNaN(factor) || factor <= 0 || factor > 1)) {
|
|
660
|
+
errors.push('rope.ropeLocalPartialRotaryFactor must be a number in (0, 1] or null');
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
if (inf.rope.ropeFrequencyBaseDim === undefined) {
|
|
664
|
+
errors.push('rope.ropeFrequencyBaseDim must be explicitly set (null when using rotary dim, or a positive even integer)');
|
|
665
|
+
} else {
|
|
666
|
+
const dim = inf.rope.ropeFrequencyBaseDim;
|
|
667
|
+
if (dim !== null && (typeof dim !== 'number' || Number.isNaN(dim) || dim <= 0 || (Math.trunc(dim) % 2) !== 0)) {
|
|
668
|
+
errors.push('rope.ropeFrequencyBaseDim must be a positive even integer or null');
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
if (inf.rope.ropeLocalFrequencyBaseDim === undefined) {
|
|
672
|
+
errors.push('rope.ropeLocalFrequencyBaseDim must be explicitly set (null when using local rotary dim, or a positive even integer)');
|
|
673
|
+
} else {
|
|
674
|
+
const dim = inf.rope.ropeLocalFrequencyBaseDim;
|
|
675
|
+
if (dim !== null && (typeof dim !== 'number' || Number.isNaN(dim) || dim <= 0 || (Math.trunc(dim) % 2) !== 0)) {
|
|
676
|
+
errors.push('rope.ropeLocalFrequencyBaseDim must be a positive even integer or null');
|
|
677
|
+
}
|
|
678
|
+
}
|
|
295
679
|
|
|
296
680
|
// Output fields - non-nullable required
|
|
297
681
|
if (inf.output.tieWordEmbeddings == null) {
|
|
@@ -391,6 +775,15 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
391
775
|
if (inf.rope.yarnOriginalMaxPos === undefined) {
|
|
392
776
|
errors.push('rope.yarnOriginalMaxPos must be explicitly set (null if not YARN)');
|
|
393
777
|
}
|
|
778
|
+
if (inf.rope.ropeLocalYarnBetaFast === undefined) {
|
|
779
|
+
errors.push('rope.ropeLocalYarnBetaFast must be explicitly set (null if not local YARN)');
|
|
780
|
+
}
|
|
781
|
+
if (inf.rope.ropeLocalYarnBetaSlow === undefined) {
|
|
782
|
+
errors.push('rope.ropeLocalYarnBetaSlow must be explicitly set (null if not local YARN)');
|
|
783
|
+
}
|
|
784
|
+
if (inf.rope.ropeLocalYarnOriginalMaxPos === undefined) {
|
|
785
|
+
errors.push('rope.ropeLocalYarnOriginalMaxPos must be explicitly set (null if not local YARN)');
|
|
786
|
+
}
|
|
394
787
|
|
|
395
788
|
if (errors.length > 0) {
|
|
396
789
|
throw new Error(
|
|
@@ -452,18 +845,234 @@ function resolveVisionConfig(rawConfig, manifest) {
|
|
|
452
845
|
);
|
|
453
846
|
return null;
|
|
454
847
|
}
|
|
848
|
+
const modelId = manifest?.modelId ?? 'unknown';
|
|
849
|
+
const resolveRequiredVisionField = (keys, label) => {
|
|
850
|
+
for (const key of keys) {
|
|
851
|
+
if (vc[key] !== undefined) {
|
|
852
|
+
return vc[key];
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
throw new Error(
|
|
856
|
+
`Manifest "${modelId}" is missing vision_config.${label}. ` +
|
|
857
|
+
'Re-convert the model with explicit vision config metadata.'
|
|
858
|
+
);
|
|
859
|
+
};
|
|
860
|
+
const resolveRequiredPositiveInteger = (keys, label) => {
|
|
861
|
+
const value = resolveRequiredVisionField(keys, label);
|
|
862
|
+
const number = Number(value);
|
|
863
|
+
if (!Number.isFinite(number) || number <= 0 || Math.floor(number) !== number) {
|
|
864
|
+
throw new Error(
|
|
865
|
+
`Manifest "${modelId}" has invalid vision_config.${label}=${JSON.stringify(value)}. ` +
|
|
866
|
+
'Expected a positive integer.'
|
|
867
|
+
);
|
|
868
|
+
}
|
|
869
|
+
return Math.trunc(number);
|
|
870
|
+
};
|
|
871
|
+
const resolveRequiredPositiveNumber = (value, label) => {
|
|
872
|
+
const number = Number(value);
|
|
873
|
+
if (!Number.isFinite(number) || number <= 0) {
|
|
874
|
+
throw new Error(
|
|
875
|
+
`Manifest "${modelId}" has invalid ${label}=${JSON.stringify(value)}. ` +
|
|
876
|
+
'Expected a positive number.'
|
|
877
|
+
);
|
|
878
|
+
}
|
|
879
|
+
return number;
|
|
880
|
+
};
|
|
881
|
+
const visionArchitecture = String(resolveRequiredVisionField(['vision_architecture'], 'vision_architecture')).trim();
|
|
882
|
+
if (visionArchitecture !== 'gemma4' && visionArchitecture !== 'qwen3vl') {
|
|
883
|
+
throw new Error(
|
|
884
|
+
`Manifest "${modelId}" has unsupported vision_config.vision_architecture="${visionArchitecture}". ` +
|
|
885
|
+
'Supported: "gemma4", "qwen3vl".'
|
|
886
|
+
);
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
if (visionArchitecture === 'gemma4') {
|
|
890
|
+
const hiddenSize = resolveRequiredPositiveInteger(['hidden_size'], 'hidden_size');
|
|
891
|
+
const numHeads = resolveRequiredPositiveInteger(['num_heads', 'num_attention_heads'], 'num_attention_heads');
|
|
892
|
+
const ropeParameters = vc.rope_parameters;
|
|
893
|
+
if (!ropeParameters || typeof ropeParameters !== 'object') {
|
|
894
|
+
throw new Error(
|
|
895
|
+
`Manifest "${modelId}" is missing vision_config.rope_parameters. ` +
|
|
896
|
+
'Re-convert the model with explicit Gemma 4 vision RoPE metadata.'
|
|
897
|
+
);
|
|
898
|
+
}
|
|
899
|
+
const hiddenActivation = String(resolveRequiredVisionField(['hidden_activation'], 'hidden_activation')).trim();
|
|
900
|
+
if (hiddenActivation !== 'gelu' && hiddenActivation !== 'gelu_pytorch_tanh') {
|
|
901
|
+
throw new Error(
|
|
902
|
+
`Manifest "${modelId}" has unsupported Gemma 4 vision hidden_activation="${hiddenActivation}". ` +
|
|
903
|
+
'Supported values: "gelu", "gelu_pytorch_tanh".'
|
|
904
|
+
);
|
|
905
|
+
}
|
|
906
|
+
if (vc.standardize === true) {
|
|
907
|
+
throw new Error(
|
|
908
|
+
`Manifest "${modelId}" enables vision_config.standardize, but Gemma 4 runtime preprocessing does not support it yet.`
|
|
909
|
+
);
|
|
910
|
+
}
|
|
911
|
+
if (vc.use_clipped_linears !== true) {
|
|
912
|
+
throw new Error(
|
|
913
|
+
`Manifest "${modelId}" requires vision_config.use_clipped_linears=true for Gemma 4 vision weights.`
|
|
914
|
+
);
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
return {
|
|
918
|
+
depth: resolveRequiredPositiveInteger(['depth', 'num_hidden_layers'], 'num_hidden_layers'),
|
|
919
|
+
hiddenSize,
|
|
920
|
+
intermediateSize: resolveRequiredPositiveInteger(['intermediate_size'], 'intermediate_size'),
|
|
921
|
+
numHeads,
|
|
922
|
+
numKeyValueHeads: resolveRequiredPositiveInteger(['num_key_value_heads'], 'num_key_value_heads'),
|
|
923
|
+
headDim: resolveRequiredPositiveInteger(['head_dim', 'global_head_dim'], 'head_dim'),
|
|
924
|
+
outHiddenSize: vc.out_hidden_size ?? vc.output_proj_dims ?? null,
|
|
925
|
+
patchSize: resolveRequiredPositiveInteger(['patch_size'], 'patch_size'),
|
|
926
|
+
poolingKernelSize: resolveRequiredPositiveInteger(['pooling_kernel_size'], 'pooling_kernel_size'),
|
|
927
|
+
spatialMergeSize: vc.spatial_merge_size ?? null,
|
|
928
|
+
temporalPatchSize: vc.temporal_patch_size ?? null,
|
|
929
|
+
positionEmbeddingSize: resolveRequiredPositiveInteger(['position_embedding_size'], 'position_embedding_size'),
|
|
930
|
+
defaultOutputLength: resolveRequiredPositiveInteger(['default_output_length'], 'default_output_length'),
|
|
931
|
+
ropeTheta: resolveRequiredPositiveNumber(ropeParameters.rope_theta, 'vision_config.rope_parameters.rope_theta'),
|
|
932
|
+
eps: resolveRequiredPositiveNumber(
|
|
933
|
+
resolveRequiredVisionField(['eps', 'rms_norm_eps'], 'rms_norm_eps'),
|
|
934
|
+
'vision_config.rms_norm_eps'
|
|
935
|
+
),
|
|
936
|
+
hiddenActivation,
|
|
937
|
+
standardize: false,
|
|
938
|
+
useClippedLinears: true,
|
|
939
|
+
deepstackVisualIndexes: [],
|
|
940
|
+
imageTokenId: rawConfig?.image_token_id ?? manifest?.image_token_id ?? null,
|
|
941
|
+
visionArchitecture,
|
|
942
|
+
softTokenBudgetTiers: Array.isArray(vc.soft_token_budget_tiers)
|
|
943
|
+
? vc.soft_token_budget_tiers.map(Number).filter((n) => Number.isFinite(n) && n > 0)
|
|
944
|
+
: [70, 140, 280, 560, 1120],
|
|
945
|
+
};
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
const hiddenSize = resolveRequiredPositiveInteger(['hidden_size'], 'hidden_size');
|
|
949
|
+
const intermediateSize = resolveRequiredPositiveInteger(['intermediate_size'], 'intermediate_size');
|
|
950
|
+
const numHeads = resolveRequiredPositiveInteger(['num_heads', 'num_attention_heads'], 'num_heads');
|
|
951
|
+
const outHiddenSize = resolveRequiredPositiveInteger(['out_hidden_size', 'output_proj_dims'], 'out_hidden_size');
|
|
952
|
+
const patchSize = resolveRequiredPositiveInteger(['patch_size'], 'patch_size');
|
|
953
|
+
const spatialMergeSize = resolveRequiredPositiveInteger(['spatial_merge_size', 'merge_size'], 'spatial_merge_size');
|
|
954
|
+
const temporalPatchSize = resolveRequiredPositiveInteger(['temporal_patch_size'], 'temporal_patch_size');
|
|
955
|
+
const eps = resolveRequiredPositiveNumber(
|
|
956
|
+
resolveRequiredVisionField(['eps', 'rms_norm_eps'], 'eps'),
|
|
957
|
+
'vision_config.eps'
|
|
958
|
+
);
|
|
959
|
+
const hiddenActivation = String(
|
|
960
|
+
resolveRequiredVisionField(['hidden_activation', 'hidden_act'], 'hidden_activation')
|
|
961
|
+
).trim();
|
|
962
|
+
const minPixels = resolveRequiredPositiveInteger(['min_pixels'], 'min_pixels');
|
|
963
|
+
const maxPixels = resolveRequiredPositiveInteger(['max_pixels'], 'max_pixels');
|
|
964
|
+
const normalization = vc.normalization;
|
|
965
|
+
if (!normalization || typeof normalization !== 'object') {
|
|
966
|
+
throw new Error(
|
|
967
|
+
`Manifest "${modelId}" is missing vision_config.normalization. ` +
|
|
968
|
+
'Re-convert the model with explicit normalization metadata.'
|
|
969
|
+
);
|
|
970
|
+
}
|
|
971
|
+
if (!Array.isArray(normalization.mean) || normalization.mean.length !== 3) {
|
|
972
|
+
throw new Error(
|
|
973
|
+
`Manifest "${modelId}" requires vision_config.normalization.mean to contain exactly 3 values.`
|
|
974
|
+
);
|
|
975
|
+
}
|
|
976
|
+
if (!Array.isArray(normalization.std) || normalization.std.length !== 3) {
|
|
977
|
+
throw new Error(
|
|
978
|
+
`Manifest "${modelId}" requires vision_config.normalization.std to contain exactly 3 values.`
|
|
979
|
+
);
|
|
980
|
+
}
|
|
455
981
|
return {
|
|
456
|
-
depth:
|
|
457
|
-
hiddenSize
|
|
458
|
-
intermediateSize
|
|
459
|
-
numHeads
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
982
|
+
depth: resolveRequiredPositiveInteger(['depth', 'num_hidden_layers'], 'depth'),
|
|
983
|
+
hiddenSize,
|
|
984
|
+
intermediateSize,
|
|
985
|
+
numHeads,
|
|
986
|
+
numKeyValueHeads: resolveRequiredPositiveInteger(['num_key_value_heads'], 'num_key_value_heads'),
|
|
987
|
+
headDim: resolveRequiredPositiveInteger(['head_dim'], 'head_dim'),
|
|
988
|
+
outHiddenSize,
|
|
989
|
+
patchSize,
|
|
990
|
+
poolingKernelSize: resolveRequiredPositiveInteger(['pooling_kernel_size'], 'pooling_kernel_size'),
|
|
991
|
+
spatialMergeSize,
|
|
992
|
+
temporalPatchSize,
|
|
993
|
+
positionEmbeddingSize: vc.position_embedding_size ?? null,
|
|
994
|
+
defaultOutputLength: vc.default_output_length ?? null,
|
|
995
|
+
ropeTheta: vc.rope_parameters?.rope_theta ?? null,
|
|
996
|
+
eps,
|
|
997
|
+
hiddenActivation,
|
|
998
|
+
standardize: vc.standardize === true,
|
|
999
|
+
useClippedLinears: vc.use_clipped_linears === true,
|
|
1000
|
+
minPixels,
|
|
1001
|
+
maxPixels,
|
|
1002
|
+
normalization,
|
|
465
1003
|
deepstackVisualIndexes: Array.isArray(vc.deepstack_visual_indexes) ? vc.deepstack_visual_indexes : [],
|
|
466
1004
|
imageTokenId: rawConfig?.image_token_id ?? manifest?.image_token_id ?? null,
|
|
1005
|
+
visionArchitecture,
|
|
1006
|
+
};
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
function resolveAudioConfig(rawConfig, manifest) {
|
|
1010
|
+
const ac = rawConfig?.audio_config ?? manifest?.config?.audio_config;
|
|
1011
|
+
if (!ac || typeof ac !== 'object') {
|
|
1012
|
+
log.debug(
|
|
1013
|
+
'Config',
|
|
1014
|
+
`Audio config not present for model "${manifest?.modelId ?? 'unknown'}"; audio pipeline disabled.`
|
|
1015
|
+
);
|
|
1016
|
+
return null;
|
|
1017
|
+
}
|
|
1018
|
+
const modelId = manifest?.modelId ?? 'unknown';
|
|
1019
|
+
const resolveRequiredPositiveInteger = (value, label) => {
|
|
1020
|
+
const number = Number(value);
|
|
1021
|
+
if (!Number.isFinite(number) || number <= 0 || Math.floor(number) !== number) {
|
|
1022
|
+
throw new Error(
|
|
1023
|
+
`Manifest "${modelId}" has invalid audio_config.${label}=${JSON.stringify(value)}. ` +
|
|
1024
|
+
'Expected a positive integer.'
|
|
1025
|
+
);
|
|
1026
|
+
}
|
|
1027
|
+
return Math.trunc(number);
|
|
1028
|
+
};
|
|
1029
|
+
const resolveRequiredPositiveNumber = (value, label) => {
|
|
1030
|
+
const number = Number(value);
|
|
1031
|
+
if (!Number.isFinite(number) || number <= 0) {
|
|
1032
|
+
throw new Error(
|
|
1033
|
+
`Manifest "${modelId}" has invalid audio_config.${label}=${JSON.stringify(value)}. ` +
|
|
1034
|
+
'Expected a positive number.'
|
|
1035
|
+
);
|
|
1036
|
+
}
|
|
1037
|
+
return number;
|
|
1038
|
+
};
|
|
1039
|
+
const audioArchitecture = String(ac.audio_architecture ?? '').trim();
|
|
1040
|
+
if (audioArchitecture !== 'gemma4') {
|
|
1041
|
+
throw new Error(
|
|
1042
|
+
`Manifest "${modelId}" has unsupported audio_config.audio_architecture="${audioArchitecture}". ` +
|
|
1043
|
+
'Supported: "gemma4".'
|
|
1044
|
+
);
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
const hiddenSize = resolveRequiredPositiveInteger(ac.hidden_size, 'hidden_size');
|
|
1048
|
+
const numAttentionHeads = resolveRequiredPositiveInteger(ac.num_attention_heads, 'num_attention_heads');
|
|
1049
|
+
const headDim = Math.trunc(hiddenSize / numAttentionHeads);
|
|
1050
|
+
|
|
1051
|
+
if (!Array.isArray(ac.subsampling_conv_channels) || ac.subsampling_conv_channels.length < 1) {
|
|
1052
|
+
throw new Error(
|
|
1053
|
+
`Manifest "${modelId}" is missing audio_config.subsampling_conv_channels array.`
|
|
1054
|
+
);
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
return {
|
|
1058
|
+
audioArchitecture,
|
|
1059
|
+
depth: resolveRequiredPositiveInteger(ac.num_hidden_layers, 'num_hidden_layers'),
|
|
1060
|
+
hiddenSize,
|
|
1061
|
+
numAttentionHeads,
|
|
1062
|
+
headDim,
|
|
1063
|
+
convKernelSize: resolveRequiredPositiveInteger(ac.conv_kernel_size, 'conv_kernel_size'),
|
|
1064
|
+
subsamplingConvChannels: ac.subsampling_conv_channels.map(Number),
|
|
1065
|
+
outputProjDims: resolveRequiredPositiveInteger(ac.output_proj_dims, 'output_proj_dims'),
|
|
1066
|
+
attentionContextLeft: resolveRequiredPositiveInteger(ac.attention_context_left, 'attention_context_left'),
|
|
1067
|
+
attentionContextRight: Number(ac.attention_context_right ?? 0),
|
|
1068
|
+
attentionChunkSize: resolveRequiredPositiveInteger(ac.attention_chunk_size, 'attention_chunk_size'),
|
|
1069
|
+
attentionLogitCap: resolveRequiredPositiveNumber(ac.attention_logit_cap, 'attention_logit_cap'),
|
|
1070
|
+
attentionInvalidLogitsValue: Number(ac.attention_invalid_logits_value ?? -1e9),
|
|
1071
|
+
residualWeight: resolveRequiredPositiveNumber(ac.residual_weight, 'residual_weight'),
|
|
1072
|
+
rmsNormEps: resolveRequiredPositiveNumber(ac.rms_norm_eps ?? 1e-6, 'rms_norm_eps'),
|
|
1073
|
+
hiddenAct: String(ac.hidden_act ?? 'silu').trim(),
|
|
1074
|
+
useClippedLinears: ac.use_clipped_linears === true,
|
|
1075
|
+
audioTokenId: rawConfig?.audio_token_id ?? manifest?.audio_token_id ?? null,
|
|
467
1076
|
};
|
|
468
1077
|
}
|
|
469
1078
|
|
|
@@ -512,8 +1121,20 @@ function parseLinearNormMode(value, sharedFlag = null, modelId = 'unknown') {
|
|
|
512
1121
|
|
|
513
1122
|
|
|
514
1123
|
export function toParsedConfigFromMerged(merged, manifest) {
|
|
515
|
-
const
|
|
516
|
-
const
|
|
1124
|
+
const mergedConfig = merged?.config ?? manifest.config ?? {};
|
|
1125
|
+
const rawConfig = mergedConfig.text_config ?? mergedConfig;
|
|
1126
|
+
const config = {
|
|
1127
|
+
...rawConfig,
|
|
1128
|
+
...(merged?.vision_config !== null && merged?.vision_config !== undefined
|
|
1129
|
+
? { vision_config: merged.vision_config }
|
|
1130
|
+
: {}),
|
|
1131
|
+
...(merged?.audio_config !== null && merged?.audio_config !== undefined
|
|
1132
|
+
? { audio_config: merged.audio_config }
|
|
1133
|
+
: {}),
|
|
1134
|
+
...(merged?.quantization_config !== null && merged?.quantization_config !== undefined
|
|
1135
|
+
? { quantization_config: merged.quantization_config }
|
|
1136
|
+
: {}),
|
|
1137
|
+
};
|
|
517
1138
|
const inf = merged.inference;
|
|
518
1139
|
|
|
519
1140
|
// Validate required fields are present (fail fast on incomplete manifests)
|
|
@@ -533,6 +1154,37 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
533
1154
|
);
|
|
534
1155
|
}
|
|
535
1156
|
const resolvedIntermediateSize = resolveIntermediateSizeForRuntime(manifest, inf, arch, merged.modelId);
|
|
1157
|
+
const archNumHeads = Number(arch.numAttentionHeads ?? arch.numHeads);
|
|
1158
|
+
const archNumKVHeads = Number(arch.numKeyValueHeads ?? arch.numKVHeads);
|
|
1159
|
+
const archHeadDim = Number(arch.headDim);
|
|
1160
|
+
const archGlobalHeadDimRaw = arch.globalHeadDim ?? null;
|
|
1161
|
+
const archGlobalHeadDim = (
|
|
1162
|
+
typeof archGlobalHeadDimRaw === 'number' && Number.isFinite(archGlobalHeadDimRaw) && archGlobalHeadDimRaw > 0
|
|
1163
|
+
)
|
|
1164
|
+
? Math.trunc(archGlobalHeadDimRaw)
|
|
1165
|
+
: null;
|
|
1166
|
+
const archNumKvSharedLayersRaw = arch.numKvSharedLayers ?? 0;
|
|
1167
|
+
const archNumKvSharedLayers = (
|
|
1168
|
+
typeof archNumKvSharedLayersRaw === 'number'
|
|
1169
|
+
&& Number.isFinite(archNumKvSharedLayersRaw)
|
|
1170
|
+
&& archNumKvSharedLayersRaw >= 0
|
|
1171
|
+
)
|
|
1172
|
+
? Math.trunc(archNumKvSharedLayersRaw)
|
|
1173
|
+
: 0;
|
|
1174
|
+
const intermediateSizes = buildPerLayerIntermediateSizes({
|
|
1175
|
+
baseIntermediateSize: resolvedIntermediateSize,
|
|
1176
|
+
numLayers: arch.numLayers,
|
|
1177
|
+
numKvSharedLayers: archNumKvSharedLayers,
|
|
1178
|
+
useDoubleWideMlp: inf.ffn.useDoubleWideMlp,
|
|
1179
|
+
modelId: merged.modelId,
|
|
1180
|
+
});
|
|
1181
|
+
const maxIntermediateSize = Math.max(...intermediateSizes);
|
|
1182
|
+
validateLayerIntermediateSizesAgainstManifest(
|
|
1183
|
+
manifest,
|
|
1184
|
+
arch.hiddenSize,
|
|
1185
|
+
intermediateSizes,
|
|
1186
|
+
merged.modelId
|
|
1187
|
+
);
|
|
536
1188
|
|
|
537
1189
|
// Compute layer types from layerPattern
|
|
538
1190
|
|
|
@@ -596,13 +1248,13 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
596
1248
|
|
|
597
1249
|
// Compute queryPreAttnScalar from manifest inference (NOT from family detection)
|
|
598
1250
|
// Manifest-first: queryPreAttnScalar is required in ManifestAttentionSchema
|
|
599
|
-
const headDim =
|
|
1251
|
+
const headDim = archHeadDim;
|
|
600
1252
|
const queryPreAttnScalar = inf.attention.queryPreAttnScalar;
|
|
601
1253
|
const causalAttention = inf.attention.causal;
|
|
602
1254
|
|
|
603
|
-
//
|
|
604
|
-
//
|
|
605
|
-
// attnScale = 1/sqrt(sqrt(headDim)) instead of the
|
|
1255
|
+
// Preserve the manifest scalar exactly. Gemma-family models legitimately use
|
|
1256
|
+
// queryPreAttnScalar=1, but sqrt(headDim) is still a known converter bug that
|
|
1257
|
+
// produces attnScale = 1/sqrt(sqrt(headDim)) instead of the intended value.
|
|
606
1258
|
if (queryPreAttnScalar != null && headDim != null
|
|
607
1259
|
&& queryPreAttnScalar !== headDim
|
|
608
1260
|
&& Math.abs(queryPreAttnScalar - Math.sqrt(headDim)) < 0.01) {
|
|
@@ -653,8 +1305,9 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
653
1305
|
const ropeLocalScale = inf.rope.ropeLocalScalingFactor;
|
|
654
1306
|
const ropeLocalScalingType = inf.rope.ropeLocalScalingType;
|
|
655
1307
|
const partialRotaryFactor = inf.rope.partialRotaryFactor;
|
|
1308
|
+
const ropeLocalPartialRotaryFactor = inf.rope.ropeLocalPartialRotaryFactor;
|
|
656
1309
|
const mropeInterleaved = inf.rope.mropeInterleaved === true;
|
|
657
|
-
const ropeInterleaved =
|
|
1310
|
+
const ropeInterleaved = inf.rope.ropeInterleaved === true;
|
|
658
1311
|
|
|
659
1312
|
if (ropeLocalScale == null && (inf.rope.ropeLocalTheta != null || inf.rope.mropeSection != null)) {
|
|
660
1313
|
throw new Error(
|
|
@@ -665,7 +1318,22 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
665
1318
|
const mropeSection = Array.isArray(inf.rope.mropeSection)
|
|
666
1319
|
? inf.rope.mropeSection.map((entry) => Math.trunc(Number(entry)))
|
|
667
1320
|
: null;
|
|
668
|
-
const ropeRotaryDim = resolveRotaryDim(
|
|
1321
|
+
const ropeRotaryDim = resolveRotaryDim(archGlobalHeadDim ?? archHeadDim, partialRotaryFactor, merged.modelId);
|
|
1322
|
+
const ropeLocalRotaryDim = resolveRotaryDim(archHeadDim, ropeLocalPartialRotaryFactor, merged.modelId);
|
|
1323
|
+
const ropeFrequencyBaseDim = resolveFrequencyBaseDim(
|
|
1324
|
+
archGlobalHeadDim ?? archHeadDim,
|
|
1325
|
+
ropeRotaryDim,
|
|
1326
|
+
inf.rope.ropeFrequencyBaseDim,
|
|
1327
|
+
merged.modelId,
|
|
1328
|
+
'rope.ropeFrequencyBaseDim'
|
|
1329
|
+
);
|
|
1330
|
+
const ropeLocalFrequencyBaseDim = resolveFrequencyBaseDim(
|
|
1331
|
+
archHeadDim,
|
|
1332
|
+
ropeLocalRotaryDim,
|
|
1333
|
+
inf.rope.ropeLocalFrequencyBaseDim,
|
|
1334
|
+
merged.modelId,
|
|
1335
|
+
'rope.ropeLocalFrequencyBaseDim'
|
|
1336
|
+
);
|
|
669
1337
|
if (mropeSection && mropeSection.some((entry) => !Number.isFinite(entry) || entry <= 0)) {
|
|
670
1338
|
throw new Error(
|
|
671
1339
|
`Manifest "${merged.modelId}" has invalid rope.mropeSection; expected positive integers.`
|
|
@@ -722,6 +1390,7 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
722
1390
|
const chatTemplateType = inf.chatTemplate.type;
|
|
723
1391
|
validateChatTemplateType(chatTemplateType, merged.modelId);
|
|
724
1392
|
const chatTemplateEnabled = inf.chatTemplate.enabled;
|
|
1393
|
+
const chatTemplateThinking = inf.chatTemplate.thinking ?? null;
|
|
725
1394
|
const parsePositiveInt = (value) => {
|
|
726
1395
|
const num = Number(value);
|
|
727
1396
|
if (!Number.isFinite(num) || num <= 0) return null;
|
|
@@ -738,15 +1407,28 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
738
1407
|
config.linear_norm_shared,
|
|
739
1408
|
merged.modelId
|
|
740
1409
|
);
|
|
1410
|
+
const hasMixedAttentionGeometry = archGlobalHeadDim != null && archGlobalHeadDim !== archHeadDim;
|
|
1411
|
+
const hasSharedKvLayers = archNumKvSharedLayers > 0;
|
|
1412
|
+
const hasExplicitLayerTypes = Array.isArray(layerTypes) && layerTypes.length === arch.numLayers;
|
|
1413
|
+
const decodeStrategy = (hasMixedAttentionGeometry || hasSharedKvLayers) && !hasExplicitLayerTypes
|
|
1414
|
+
? 'replay_prefill'
|
|
1415
|
+
: 'incremental';
|
|
1416
|
+
const perLayerInputsSession = resolvePerLayerInputsSession(inf, merged.modelId);
|
|
741
1417
|
|
|
742
1418
|
return {
|
|
743
1419
|
numLayers: arch.numLayers,
|
|
744
1420
|
hiddenSize: arch.hiddenSize,
|
|
745
1421
|
intermediateSize: resolvedIntermediateSize,
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
1422
|
+
intermediateSizes,
|
|
1423
|
+
maxIntermediateSize,
|
|
1424
|
+
numHeads: archNumHeads,
|
|
1425
|
+
numKVHeads: archNumKVHeads,
|
|
1426
|
+
headDim: archHeadDim,
|
|
1427
|
+
globalHeadDim: archGlobalHeadDim,
|
|
749
1428
|
vocabSize: arch.vocabSize,
|
|
1429
|
+
hiddenSizePerLayerInput: arch.hiddenSizePerLayerInput ?? null,
|
|
1430
|
+
vocabSizePerLayerInput: arch.vocabSizePerLayerInput ?? null,
|
|
1431
|
+
numKvSharedLayers: archNumKvSharedLayers,
|
|
750
1432
|
maxSeqLen: arch.maxSeqLen,
|
|
751
1433
|
useMoE,
|
|
752
1434
|
numExperts,
|
|
@@ -756,10 +1438,14 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
756
1438
|
ropeTheta: inf.rope.ropeTheta,
|
|
757
1439
|
ropeLocalTheta: inf.rope.ropeLocalTheta,
|
|
758
1440
|
ropeRotaryDim,
|
|
1441
|
+
ropeLocalRotaryDim,
|
|
1442
|
+
ropeFrequencyBaseDim,
|
|
1443
|
+
ropeLocalFrequencyBaseDim,
|
|
759
1444
|
ropeInterleaved,
|
|
760
1445
|
mropeInterleaved,
|
|
761
1446
|
mropeSection,
|
|
762
1447
|
partialRotaryFactor,
|
|
1448
|
+
ropeLocalPartialRotaryFactor,
|
|
763
1449
|
ropeScale,
|
|
764
1450
|
ropeLocalScale,
|
|
765
1451
|
ropeScalingType,
|
|
@@ -779,6 +1465,7 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
779
1465
|
embeddingVocabSize: inf.output.embeddingVocabSize,
|
|
780
1466
|
embeddingPostprocessor,
|
|
781
1467
|
hiddenActivation,
|
|
1468
|
+
useDoubleWideMlp: inf.ffn.useDoubleWideMlp,
|
|
782
1469
|
swigluLimit: inf.ffn.swigluLimit,
|
|
783
1470
|
stopTokenIds,
|
|
784
1471
|
layerTypes,
|
|
@@ -793,13 +1480,18 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
793
1480
|
finalLogitSoftcapping: inf.output.finalLogitSoftcapping,
|
|
794
1481
|
attnLogitSoftcapping: inf.attention.attnLogitSoftcapping,
|
|
795
1482
|
queryKeyNorm: inf.attention.queryKeyNorm,
|
|
1483
|
+
valueNorm: inf.attention.valueNorm,
|
|
796
1484
|
attentionOutputGate: inf.attention.attentionOutputGate === true,
|
|
797
1485
|
queryPreAttnScalar,
|
|
798
1486
|
layerPipeline: inf.pipeline ?? null,
|
|
799
1487
|
chatTemplateType,
|
|
800
1488
|
chatTemplateEnabled,
|
|
1489
|
+
chatTemplateThinking,
|
|
1490
|
+
decodeStrategy,
|
|
1491
|
+
perLayerInputsSession,
|
|
801
1492
|
kernelPath: null,
|
|
802
1493
|
visionConfig: resolveVisionConfig(config, manifest),
|
|
1494
|
+
audioConfig: resolveAudioConfig(config, manifest),
|
|
803
1495
|
};
|
|
804
1496
|
}
|
|
805
1497
|
|
|
@@ -813,6 +1505,8 @@ export function parseModelConfigFromManifest(manifest, runtimeOverrides) {
|
|
|
813
1505
|
modelId: manifest.modelId ?? 'unknown',
|
|
814
1506
|
inference: manifest.inference,
|
|
815
1507
|
architecture: manifest.architecture,
|
|
1508
|
+
config: manifest.config ?? null,
|
|
1509
|
+
quantization_config: manifest.quantization_config ?? null,
|
|
816
1510
|
},
|
|
817
1511
|
runtimeOverrides
|
|
818
1512
|
);
|