doppler-gpu 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -577
- package/LICENSE +21 -201
- package/README.md +165 -222
- package/models/catalog.json +241 -33
- package/package.json +99 -86
- package/src/capsule-runtime.d.ts +23 -0
- package/src/capsule-runtime.js +23 -0
- package/src/capsule.d.ts +8 -0
- package/src/capsule.js +4 -0
- package/src/cli/argument-contract.js +200 -0
- package/src/cli/browser-report-output.js +24 -0
- package/src/cli/cli-model-resolution.js +2 -2
- package/src/cli/cli-output.js +10 -0
- package/src/cli/commands/bundle.js +828 -0
- package/src/cli/doppler-cli.js +50 -1457
- package/src/cli/doppler-serve.js +18 -244
- package/src/cli/output/manifest.js +368 -0
- package/src/cli/runtime-profiles.js +1 -1
- package/src/cli/serve/capsule-handler.d.ts +33 -0
- package/src/cli/serve/capsule-handler.js +105 -0
- package/src/cli/serve/capsule-http.js +102 -0
- package/src/cli/serve/receipt.js +269 -0
- package/src/client/capsule-host.browser.js +1 -0
- package/src/client/capsule-host.d.ts +3 -0
- package/src/client/capsule-host.js +1 -0
- package/src/client/doppler-api.browser.js +42 -0
- package/src/client/doppler-api.d.ts +36 -1
- package/src/client/doppler-api.js +40 -1
- package/src/client/doppler-registry.js +15 -7
- package/src/client/electron/index.d.ts +4 -0
- package/src/client/electron/index.js +13 -0
- package/src/client/electron/ipc-contract.d.ts +9 -0
- package/src/client/electron/ipc-contract.js +56 -0
- package/src/client/electron/release-state.d.ts +53 -0
- package/src/client/electron/release-state.js +425 -0
- package/src/client/electron/renderer-runtime.d.ts +19 -0
- package/src/client/electron/renderer-runtime.js +104 -0
- package/src/client/failure-taxonomy.js +1 -28
- package/src/client/fault-injection.js +0 -8
- package/src/client/inspection.js +1 -1
- package/src/client/model-host/index.d.ts +140 -0
- package/src/client/model-host/index.js +548 -0
- package/src/client/model-host/model-evidence.js +255 -0
- package/src/client/model-host/model-session.d.ts +322 -0
- package/src/client/{runtime → model-host}/model-session.js +190 -148
- package/src/client/provider.d.ts +18 -2
- package/src/client/provider.js +26 -3
- package/src/client/receipt.js +30 -72
- package/src/client/runtime/capsule-acquisition.d.ts +18 -0
- package/src/client/runtime/capsule-acquisition.js +103 -0
- package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
- package/src/client/runtime/capsule-adapter-execution.js +63 -0
- package/src/client/runtime/capsule-artifact-source.js +62 -0
- package/src/client/runtime/capsule-embedding.d.ts +29 -0
- package/src/client/runtime/capsule-embedding.js +76 -0
- package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
- package/src/client/runtime/capsule-forecast-program.js +85 -0
- package/src/client/runtime/capsule-forecast.d.ts +17 -0
- package/src/client/runtime/capsule-forecast.js +31 -0
- package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
- package/src/client/runtime/capsule-operation-adapters.js +68 -0
- package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
- package/src/client/runtime/capsule-operation-executor.js +100 -0
- package/src/client/runtime/capsule-program-adapter.js +151 -0
- package/src/client/runtime/capsule-release-authorization.js +25 -0
- package/src/client/runtime/capsule-rerank.d.ts +50 -0
- package/src/client/runtime/capsule-rerank.js +105 -0
- package/src/client/runtime/capsule-session-execution.js +85 -0
- package/src/client/runtime/command-executor.d.ts +15 -0
- package/src/client/runtime/command-executor.js +128 -0
- package/src/client/runtime/composition-root.d.ts +74 -0
- package/src/client/runtime/composition-root.js +344 -0
- package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
- package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
- package/src/client/runtime/index.d.ts +1 -117
- package/src/client/runtime/index.js +1 -433
- package/src/client/runtime/lora.d.ts +14 -0
- package/src/client/runtime/lora.js +19 -1
- package/src/client/runtime/model-manager.js +12 -0
- package/src/client/runtime/model-rerank.js +30 -0
- package/src/client/runtime/model-session.d.ts +1 -221
- package/src/client/runtime/model-source.d.ts +9 -0
- package/src/client/runtime/model-source.js +46 -17
- package/src/client/runtime/node-quickstart-cache.js +1 -1
- package/src/client/runtime/resolution-policy.d.ts +20 -0
- package/src/client/runtime/resolution-policy.js +84 -0
- package/src/client/runtime/resource-binder.d.ts +24 -0
- package/src/client/runtime/resource-binder.js +135 -0
- package/src/client/runtime/scoped-session.d.ts +10 -1
- package/src/client/runtime/scoped-session.js +20 -1
- package/src/client/runtime/session-controller.d.ts +30 -0
- package/src/client/runtime/session-controller.js +170 -0
- package/src/client/runtime/source-runtime.js +1 -1
- package/src/client/runtime/target-selector.d.ts +9 -0
- package/src/client/runtime/target-selector.js +2 -0
- package/src/client/runtime/types.d.ts +1 -0
- package/src/client/runtime/verified-capsule-artifact-store.js +72 -0
- package/src/client/wrap-pipeline-handle.js +3 -9
- package/src/config/backward-registry-loader.js +1 -1
- package/src/config/capsule-adapter-policy.js +30 -0
- package/src/config/capsule-adapters.d.ts +21 -0
- package/src/config/capsule-adapters.js +40 -0
- package/src/config/capsule-adapters.json +9 -0
- package/src/config/capsule-loading.d.ts +2 -0
- package/src/config/capsule-loading.js +14 -0
- package/src/config/capsule-loading.schema.json +11 -0
- package/src/config/capsule-operation.d.ts +19 -0
- package/src/config/capsule-operation.js +58 -0
- package/src/config/capsule-operations.json +18 -0
- package/src/config/capsule-release-contract.d.ts +66 -0
- package/src/config/capsule-release-contract.js +197 -0
- package/src/config/capsule-release-events.d.ts +53 -0
- package/src/config/capsule-release-events.js +172 -0
- package/src/config/capsule-sequence-receipt.js +19 -0
- package/src/config/capsule-serve.d.ts +9 -0
- package/src/config/capsule-serve.js +28 -0
- package/src/config/capsule-serve.schema.json +14 -0
- package/src/config/capsule-signature.d.ts +6 -0
- package/src/config/capsule-signature.js +50 -0
- package/src/config/capsule-v2.d.ts +70 -0
- package/src/config/capsule-v2.js +431 -0
- package/src/config/capsule-v3.d.ts +19 -0
- package/src/config/capsule-v3.js +72 -0
- package/src/config/capsule.d.ts +22 -0
- package/src/config/capsule.js +52 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +41 -1
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +1 -1
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +59 -7
- package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
- package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
- package/src/config/dtype-transition-contract.js +40 -0
- package/src/config/embedding-contract.js +26 -0
- package/src/config/embedding-reference.js +85 -0
- package/src/config/evidence/source-intake-policy.json +1 -1
- package/src/config/execution-contract-check.d.ts +16 -73
- package/src/config/execution-contract-check.js +7 -429
- package/src/config/forecast-manifest.js +56 -0
- package/src/config/heterogeneous-attention-contract.js +46 -0
- package/src/config/initial-execution-identity.d.ts +58 -0
- package/src/config/initial-execution-identity.js +331 -0
- package/src/config/kernel-path-loader.d.ts +10 -0
- package/src/config/kernel-path-loader.js +23 -8
- package/src/config/kernel-registry-contract.d.ts +25 -0
- package/src/config/kernel-registry-contract.js +46 -0
- package/src/config/kernels/kernel-ref-digests.js +50 -2
- package/src/config/kernels/registry.js +1 -1
- package/src/config/kernels/registry.json +13592 -11377
- package/src/config/lora-layouts.d.ts +10 -0
- package/src/config/lora-layouts.js +17 -0
- package/src/config/lora-layouts.json +9 -0
- package/src/config/manifest-inference-validation.js +575 -0
- package/src/config/merge/execution-patches.js +12 -0
- package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
- package/src/config/merge/value-selection.js +38 -0
- package/src/config/merge.js +43 -11
- package/src/config/model-ir-v2.d.ts +57 -0
- package/src/config/model-ir-v2.js +258 -0
- package/src/config/model-ir.d.ts +49 -0
- package/src/config/model-ir.js +180 -0
- package/src/config/param-validator.js +1 -1
- package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
- package/src/config/platforms/loader.js +2 -1
- package/src/config/production-release-evidence.d.ts +23 -0
- package/src/config/production-release-evidence.js +423 -0
- package/src/config/production-release.js +321 -0
- package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
- package/src/config/required-inference-fields-contract-check.js +2 -2
- package/src/config/rerank-reference.d.ts +28 -0
- package/src/config/rerank-reference.js +89 -0
- package/src/config/revocation-policy.d.ts +65 -0
- package/src/config/revocation-policy.js +290 -0
- package/src/config/revocation-registry.json +11 -0
- package/src/config/revocation-updates.d.ts +68 -0
- package/src/config/revocation-updates.js +440 -0
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
- package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
- package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
- package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
- package/src/config/runtime-merge.js +1 -1
- package/src/config/runtime.js +1 -1
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/conversion-report.schema.d.ts +3 -0
- package/src/config/schema/conversion-report.schema.js +29 -0
- package/src/config/schema/converter.schema.d.ts +1 -0
- package/src/config/schema/debug.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +5 -5
- package/src/config/schema/emulation.schema.js +1 -1
- package/src/config/schema/execution-v1.schema.d.ts +3 -0
- package/src/config/schema/execution-v1.schema.js +19 -0
- package/src/config/schema/harness.schema.d.ts +13 -0
- package/src/config/schema/harness.schema.js +6 -0
- package/src/config/schema/inference.schema.d.ts +21 -2
- package/src/config/schema/kernel-thresholds.schema.js +1 -1
- package/src/config/schema/manifest.schema.d.ts +26 -7
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/program-bundle.schema.d.ts +52 -6
- package/src/config/schema/program-bundle.schema.js +145 -10
- package/src/config/sequence-reference.d.ts +26 -0
- package/src/config/sequence-reference.js +100 -0
- package/src/config/target-plan.d.ts +108 -0
- package/src/config/target-plan.js +411 -0
- package/src/config/training-defaults.js +1 -1
- package/src/config/transforms/capability-transform-resolver.js +4 -105
- package/src/config/transforms/execution-graph/dtype.js +591 -0
- package/src/config/transforms/execution-graph/fusion.js +436 -0
- package/src/config/transforms/execution-graph/validation.js +414 -0
- package/src/config/transforms/execution-graph-transforms.js +13 -1419
- package/src/converter/artifact-identity.js +475 -0
- package/src/converter/conversion-plan.js +1 -1
- package/src/converter/core.js +94 -2096
- package/src/converter/forge-candidate-evaluation.js +271 -0
- package/src/converter/forge-qualification.js +162 -0
- package/src/converter/forge-stages.js +798 -0
- package/src/converter/manifest-builder.js +683 -0
- package/src/converter/quantization-info.js +4 -1
- package/src/converter/quantizer.js +58 -9
- package/src/converter/shard-packer.d.ts +1 -1
- package/src/converter/shard-packer.js +2 -2
- package/src/converter/tensor-transform.js +640 -0
- package/src/converter/tokenizer-contract.js +243 -0
- package/src/converter/tokenizer-manifest.js +243 -0
- package/src/debug/history.js +0 -8
- package/src/debug/index.js +0 -10
- package/src/debug/tensor.js +1 -1
- package/src/debug/trace.js +0 -7
- package/src/experimental/adapters/adapter-manager.js +2 -0
- package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
- package/src/experimental/adapters/adapter-manifest.js +11 -0
- package/src/experimental/adapters/artifact-contract.d.ts +2 -2
- package/src/experimental/adapters/artifact-contract.js +5 -2
- package/src/experimental/adapters/lora-loader.d.ts +10 -1
- package/src/experimental/adapters/lora-loader.js +110 -48
- package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
- package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
- package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
- package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
- package/src/experimental/browser/browser-converter.js +11 -11
- package/src/experimental/diffusion/index.d.ts +1 -1
- package/src/experimental/diffusion/index.js +1 -1
- package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
- package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
- package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
- package/src/experimental/diffusion/vae.js +596 -0
- package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
- package/src/experimental/distribution/shard-delivery/retry.js +392 -0
- package/src/experimental/distribution/shard-delivery/transport.js +831 -0
- package/src/experimental/distribution/shard-delivery.js +4 -1709
- package/src/experimental/energy/index.d.ts +2 -2
- package/src/experimental/energy/index.js +2 -2
- package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
- package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
- package/src/experimental/logits/cpu-output.d.ts +27 -0
- package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
- package/src/experimental/logits/cpu.d.ts +87 -0
- package/src/experimental/logits/cpu.js +95 -0
- package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
- package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
- package/src/experimental/orchestration/index.d.ts +1 -1
- package/src/experimental/orchestration/index.js +1 -1
- package/src/experimental/training/artifacts.js +2 -2
- package/src/experimental/training/checkpoint.js +2 -2
- package/src/experimental/training/distillation/artifacts.js +1 -1
- package/src/experimental/training/distillation/dataset.js +2 -2
- package/src/experimental/training/distillation/eval.js +1 -1
- package/src/experimental/training/distillation/runtime.js +2 -2
- package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
- package/src/experimental/training/distillation/student-fixture/model.js +353 -0
- package/src/experimental/training/distillation/student-fixture.js +2 -469
- package/src/experimental/training/lora/checkpoint.js +746 -0
- package/src/experimental/training/lora/execution.js +550 -0
- package/src/experimental/training/lora/plan.js +387 -0
- package/src/experimental/training/lora/recovery.js +418 -0
- package/src/experimental/training/lora-pipeline.js +7 -1949
- package/src/experimental/training/operator-artifacts.js +2 -2
- package/src/experimental/training/operator-command.js +4 -4
- package/src/experimental/training/runner/evaluation.js +62 -0
- package/src/experimental/training/runner/lifecycle.js +257 -0
- package/src/experimental/training/runner.js +5 -268
- package/src/experimental/training/suite/execution.js +879 -0
- package/src/experimental/training/suite/plan.js +855 -0
- package/src/experimental/training/suite.js +5 -1659
- package/src/experimental/training/workloads.d.ts +5 -5
- package/src/experimental/training/workloads.js +6 -6
- package/src/experimental/turboquant/codebook.js +470 -0
- package/src/formats/quantization-constants.d.ts +11 -0
- package/src/formats/quantization-constants.js +11 -0
- package/src/formats/rdrr/integrity.js +1 -1
- package/src/formats/rdrr/lowerings.js +0 -29
- package/src/formats/rdrr/merkle.js +1 -1
- package/src/formats/rdrr/tensor-config-validator.js +25 -5
- package/src/formats/rdrr/validation.js +3 -3
- package/src/gpu/command-recorder.d.ts +10 -1
- package/src/gpu/command-recorder.js +58 -14
- package/src/gpu/constant-buffer.js +29 -0
- package/src/gpu/device-state.js +36 -0
- package/src/gpu/device.js +6 -41
- package/src/gpu/kernel-runtime.js +2 -1
- package/src/gpu/kernels/activation-static-qdq.js +1 -1
- package/src/gpu/kernels/append_embedding.wgsl +14 -0
- package/src/gpu/kernels/attention/executors.js +960 -0
- package/src/gpu/kernels/attention/plan.js +472 -0
- package/src/gpu/kernels/attention.d.ts +4 -0
- package/src/gpu/kernels/attention.js +145 -1545
- package/src/gpu/kernels/attention_relative.wgsl +59 -0
- package/src/gpu/kernels/backward/adam.js +2 -1
- package/src/gpu/kernels/backward/attention_backward.js +1 -1
- package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
- package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
- package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
- package/src/gpu/kernels/backward/embed_backward.js +2 -1
- package/src/gpu/kernels/backward/executor.d.ts +34 -0
- package/src/gpu/kernels/backward/executor.js +158 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
- package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/gelu_backward.js +1 -1
- package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
- package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
- package/src/gpu/kernels/backward/matmul.js +133 -0
- package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
- package/src/gpu/kernels/backward/matmul_backward.js +1 -1
- package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
- package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
- package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/rope_backward.js +2 -1
- package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/scale_backward.js +2 -1
- package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/silu_backward.js +1 -1
- package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/softmax_backward.js +1 -1
- package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
- package/src/gpu/kernels/backward/utils.js +2 -392
- package/src/gpu/kernels/bind-group-validation.js +16 -0
- package/src/gpu/kernels/cast.js +2 -1
- package/src/gpu/kernels/check-finiteness.js +13 -49
- package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
- package/src/gpu/kernels/check-stop.js +14 -33
- package/src/gpu/kernels/check_finiteness.wgsl +33 -0
- package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
- package/src/gpu/kernels/check_stop.wgsl +20 -0
- package/src/gpu/kernels/clamp.js +1 -1
- package/src/gpu/kernels/conv2d.js +1 -1
- package/src/gpu/kernels/cross_entropy_loss.js +1 -1
- package/src/gpu/kernels/depthwise_conv1d.js +1 -1
- package/src/gpu/kernels/depthwise_conv2d.js +1 -1
- package/src/gpu/kernels/dequant.js +7 -2
- package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
- package/src/gpu/kernels/dispatch.js +12 -1
- package/src/gpu/kernels/embedding-pool.js +72 -0
- package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
- package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
- package/src/gpu/kernels/energy.js +2 -1
- package/src/gpu/kernels/forecast_output.wgsl +17 -0
- package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
- package/src/gpu/kernels/fused_ffn.js +10 -220
- package/src/gpu/kernels/fused_matmul_residual.js +2 -1
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
- package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
- package/src/gpu/kernels/gated-short-conv.js +11 -69
- package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
- package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
- package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
- package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
- package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
- package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
- package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
- package/src/gpu/kernels/gather.d.ts +1 -1
- package/src/gpu/kernels/gather.js +30 -6
- package/src/gpu/kernels/gather_bf16.wgsl +69 -0
- package/src/gpu/kernels/gelu.js +1 -1
- package/src/gpu/kernels/gemma4-route-expert.js +1 -1
- package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
- package/src/gpu/kernels/groupnorm.js +1 -1
- package/src/gpu/kernels/index.d.ts +38 -5
- package/src/gpu/kernels/index.js +18 -4
- package/src/gpu/kernels/instance_stats.wgsl +26 -0
- package/src/gpu/kernels/kernel-base.js +1 -1
- package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
- package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
- package/src/gpu/kernels/kernel-configs.d.ts +10 -61
- package/src/gpu/kernels/kernel-configs.js +26 -54
- package/src/gpu/kernels/kernel-execution.js +141 -0
- package/src/gpu/kernels/kv-cache-write.js +4 -4
- package/src/gpu/kernels/kv-quantize.js +4 -1
- package/src/gpu/kernels/l2-normalize.js +36 -0
- package/src/gpu/kernels/l2_normalize.wgsl +53 -0
- package/src/gpu/kernels/layernorm.d.ts +4 -4
- package/src/gpu/kernels/layernorm.js +44 -7
- package/src/gpu/kernels/linear-attention-core.js +29 -449
- package/src/gpu/kernels/linear_attention.js +1 -1
- package/src/gpu/kernels/lm-head-argmax.js +2 -2
- package/src/gpu/kernels/lm-head-select-logits.js +2 -2
- package/src/gpu/kernels/logit-finalize.js +103 -0
- package/src/gpu/kernels/logit-merge.js +13 -125
- package/src/gpu/kernels/logit-scatter.js +40 -0
- package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
- package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
- package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
- package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
- package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
- package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
- package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
- package/src/gpu/kernels/matmul-dispatch.js +6 -1
- package/src/gpu/kernels/matmul-selection.js +3 -2
- package/src/gpu/kernels/matmul.js +4 -15
- package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
- package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
- package/src/gpu/kernels/modulate.js +1 -1
- package/src/gpu/kernels/moe.js +3 -1
- package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
- package/src/gpu/kernels/patch_context.wgsl +29 -0
- package/src/gpu/kernels/pipeline-cache.js +3 -1
- package/src/gpu/kernels/pixel_shuffle.js +1 -1
- package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
- package/src/gpu/kernels/relu.js +1 -1
- package/src/gpu/kernels/repeat_channels.js +1 -1
- package/src/gpu/kernels/residual.js +2 -2
- package/src/gpu/kernels/rmsnorm-pair.js +1 -1
- package/src/gpu/kernels/rmsnorm-qk.js +1 -1
- package/src/gpu/kernels/rmsnorm-stats.js +14 -113
- package/src/gpu/kernels/rmsnorm.js +1 -1
- package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
- package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
- package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
- package/src/gpu/kernels/rope-precompute.js +160 -0
- package/src/gpu/kernels/rope-qk.js +1 -1
- package/src/gpu/kernels/rope.js +2 -2
- package/src/gpu/kernels/rope_precompute.wgsl +74 -0
- package/src/gpu/kernels/sample.js +5 -1
- package/src/gpu/kernels/scale.js +1 -1
- package/src/gpu/kernels/shader-cache.js +8 -4
- package/src/gpu/kernels/shader-source-scope.js +68 -0
- package/src/gpu/kernels/silu.js +2 -1
- package/src/gpu/kernels/soft-embedding.js +3 -1
- package/src/gpu/kernels/softmax.js +4 -2
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
- package/src/gpu/kernels/split_qg.js +1 -1
- package/src/gpu/kernels/split_qkv.d.ts +2 -0
- package/src/gpu/kernels/split_qkv.js +18 -3
- package/src/gpu/kernels/split_qkv.wgsl +5 -2
- package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
- package/src/gpu/kernels/submit_probe.wgsl +6 -0
- package/src/gpu/kernels/transpose.js +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +4 -535
- package/src/gpu/kernels/uniform-utils.js +7 -2
- package/src/gpu/kernels/upsample2d.js +1 -1
- package/src/gpu/kernels/utils.js +9 -330
- package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
- package/src/gpu/kernels/vision-average-pool.js +68 -0
- package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
- package/src/gpu/kernels/vision-patch-embed.js +117 -0
- package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
- package/src/gpu/kernels/vision-position-embedding.js +79 -0
- package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
- package/src/gpu/kernels/vision-rope-2d.js +73 -0
- package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
- package/src/gpu/kernels/vision-spatial-merge.js +100 -0
- package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
- package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
- package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
- package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
- package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
- package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
- package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
- package/src/gpu/readback-buffer.js +20 -0
- package/src/gpu/submit-probe.js +2 -26
- package/src/gpu/weight-buffer.d.ts +1 -0
- package/src/gpu/weight-buffer.js +7 -0
- package/src/index.d.ts +10 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness/benchmark-execution.js +872 -0
- package/src/inference/browser-harness/benchmark-rerank.js +284 -0
- package/src/inference/browser-harness/decode-diagnostics.js +241 -0
- package/src/inference/browser-harness/diffusion-performance.js +102 -0
- package/src/inference/browser-harness/inference-execution.js +543 -0
- package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
- package/src/inference/browser-harness/model-resolution.js +137 -0
- package/src/inference/browser-harness/report.js +338 -0
- package/src/inference/browser-harness/request.js +263 -0
- package/src/inference/browser-harness/runtime-config.d.ts +25 -0
- package/src/inference/browser-harness/runtime-config.js +255 -0
- package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
- package/src/inference/browser-harness/runtime-isolation.js +45 -0
- package/src/inference/browser-harness/runtime-report.d.ts +26 -0
- package/src/inference/browser-harness/runtime-report.js +117 -0
- package/src/inference/browser-harness/suite-summary.js +39 -0
- package/src/inference/browser-harness/text-evidence.js +694 -0
- package/src/inference/browser-harness/text-execution.js +757 -0
- package/src/inference/browser-harness/text-input.js +725 -0
- package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
- package/src/inference/browser-harness-contract-helpers.js +1 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
- package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
- package/src/inference/browser-harness-runtime-helpers.js +3 -423
- package/src/inference/browser-harness-text-helpers.js +31 -2199
- package/src/inference/browser-harness.d.ts +15 -2
- package/src/inference/browser-harness.js +38 -1952
- package/src/inference/kv-cache/base.js +67 -381
- package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
- package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
- package/src/inference/kv-cache/quantized.d.ts +2 -2
- package/src/inference/kv-cache/quantized.js +4 -8
- package/src/inference/kv-cache/update.js +309 -0
- package/src/inference/model-checkpoint-evidence.d.ts +75 -0
- package/src/inference/model-checkpoint-evidence.js +264 -0
- package/src/inference/moe-router.js +36 -216
- package/src/inference/multi-model-network.js +5 -341
- package/src/inference/pipelines/audio/index.js +1 -20
- package/src/inference/pipelines/audio/mel.js +0 -38
- package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
- package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
- package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
- package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
- package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
- package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
- package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
- package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
- package/src/inference/pipelines/factory.js +14 -3
- package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +0 -6
- package/src/inference/pipelines/text/abort-contract.js +22 -0
- package/src/inference/pipelines/text/attention/attn-config.js +1 -16
- package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
- package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
- package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
- package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
- package/src/inference/pipelines/text/attention/interpreter.js +89 -439
- package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
- package/src/inference/pipelines/text/attention/plan.js +17 -0
- package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
- package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
- package/src/inference/pipelines/text/attention/projections.js +20 -537
- package/src/inference/pipelines/text/attention/query-transform.js +11 -0
- package/src/inference/pipelines/text/attention/record.js +0 -6
- package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
- package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
- package/src/inference/pipelines/text/attention/run.js +0 -6
- package/src/inference/pipelines/text/attention/types.js +3 -31
- package/src/inference/pipelines/text/chat-format.d.ts +3 -0
- package/src/inference/pipelines/text/chat-format.js +42 -0
- package/src/inference/pipelines/text/config/normalization.js +729 -0
- package/src/inference/pipelines/text/config/validation.js +9 -0
- package/src/inference/pipelines/text/config.d.ts +12 -1
- package/src/inference/pipelines/text/config.js +14 -1228
- package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
- package/src/inference/pipelines/text/debug-utils/index.js +3 -8
- package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
- package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
- package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +1 -68
- package/src/inference/pipelines/text/embed.js +178 -241
- package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
- package/src/inference/pipelines/text/embedding-contract.js +21 -0
- package/src/inference/pipelines/text/embedding-extraction.js +225 -0
- package/src/inference/pipelines/text/embedding-normalization.js +39 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
- package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
- package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
- package/src/inference/pipelines/text/execution-v1.js +7 -283
- package/src/inference/pipelines/text/execution.js +517 -0
- package/src/inference/pipelines/text/ffn/dense-executor.js +718 -0
- package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
- package/src/inference/pipelines/text/ffn/dense.js +4 -968
- package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
- package/src/inference/pipelines/text/ffn/standard.js +2 -1
- package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
- package/src/inference/pipelines/text/generator/decode-runtime.js +991 -0
- package/src/inference/pipelines/text/generator/decode.js +952 -0
- package/src/inference/pipelines/text/generator/diffusion.js +607 -0
- package/src/inference/pipelines/text/generator/logits-config.js +63 -0
- package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
- package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
- package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
- package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
- package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
- package/src/inference/pipelines/text/generator/recovery.js +310 -0
- package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
- package/src/inference/pipelines/text/generator/text.js +448 -0
- package/src/inference/pipelines/text/generator/training.js +752 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
- package/src/inference/pipelines/text/generator-logits-observation.js +45 -0
- package/src/inference/pipelines/text/generator-runtime.js +5 -135
- package/src/inference/pipelines/text/generator-steps.js +30 -972
- package/src/inference/pipelines/text/generator.js +165 -3544
- package/src/inference/pipelines/text/image-transcription.js +304 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
- package/src/inference/pipelines/text/init-rope.js +37 -189
- package/src/inference/pipelines/text/init.d.ts +8 -3
- package/src/inference/pipelines/text/init.js +6 -216
- package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
- package/src/inference/pipelines/text/layer-input-execution.js +435 -0
- package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
- package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
- package/src/inference/pipelines/text/layer-plan.js +45 -3
- package/src/inference/pipelines/text/layer.js +52 -522
- package/src/inference/pipelines/text/lifecycle.js +839 -0
- package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
- package/src/inference/pipelines/text/linear-attention.js +7 -537
- package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
- package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
- package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
- package/src/inference/pipelines/text/logits/cpu.js +8 -95
- package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
- package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
- package/src/inference/pipelines/text/logits/gpu.js +116 -579
- package/src/inference/pipelines/text/logits/index.d.ts +2 -1
- package/src/inference/pipelines/text/logits/index.js +62 -102
- package/src/inference/pipelines/text/logits/output-transform.js +47 -0
- package/src/inference/pipelines/text/logits/plan.js +184 -0
- package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
- package/src/inference/pipelines/text/logits/readback.js +9 -0
- package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/lora-apply.js +12 -4
- package/src/inference/pipelines/text/lora-types.d.ts +12 -0
- package/src/inference/pipelines/text/modality-token-contract.js +115 -0
- package/src/inference/pipelines/text/model-load.js +0 -20
- package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
- package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
- package/src/inference/pipelines/text/moe/plan.js +47 -0
- package/src/inference/pipelines/text/moe-cpu.js +9 -135
- package/src/inference/pipelines/text/moe-gpu.js +21 -545
- package/src/inference/pipelines/text/normalization-contract.js +17 -0
- package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
- package/src/inference/pipelines/text/per-layer/plan.js +362 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
- package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
- package/src/inference/pipelines/text/probes.js +20 -0
- package/src/inference/pipelines/text/state.d.ts +8 -0
- package/src/inference/pipelines/text/state.js +1 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
- package/src/inference/pipelines/text/types.d.ts +4 -0
- package/src/inference/pipelines/text/weight-loading-config.js +194 -0
- package/src/inference/pipelines/text.d.ts +2 -2
- package/src/inference/pipelines/text.js +46 -1633
- package/src/inference/pipelines/video/frame-sampler.js +0 -13
- package/src/inference/pipelines/video/index.js +0 -21
- package/src/inference/pipelines/vision/attention.js +59 -0
- package/src/inference/pipelines/vision/encoder.js +254 -369
- package/src/inference/pipelines/vision/gemma4.js +204 -318
- package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
- package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
- package/src/inference/pipelines/vision/glmocr.js +557 -0
- package/src/inference/pipelines/vision/image-preprocess.js +0 -27
- package/src/inference/pipelines/vision/index.js +8 -27
- package/src/inference/pipelines/vision/ops.js +11 -33
- package/src/inference/pipelines/vision/patch-embed.js +51 -145
- package/src/inference/pipelines/vision/spatial-merge.js +78 -0
- package/src/inference/pipelines/vision/weight-loading.js +79 -0
- package/src/inference/refactor-receipt.js +2 -2
- package/src/inference/rerank.js +182 -0
- package/src/inference/tokenizer.js +1 -1
- package/src/inference/tokenizers/behavior-flags.js +6 -3
- package/src/inference/tokenizers/bundled/decoders.js +133 -0
- package/src/inference/tokenizers/bundled/registry.js +216 -0
- package/src/inference/tokenizers/bundled.js +23 -353
- package/src/loader/doppler-loader.d.ts +4 -0
- package/src/loader/doppler-loader.js +119 -930
- package/src/loader/embedding-limit-preflight.js +11 -4
- package/src/loader/embedding-loader.js +1 -3
- package/src/loader/gpu-tensor-materialization.js +20 -0
- package/src/loader/layer-loader.js +15 -119
- package/src/loader/layer-tensor-aliases.js +99 -0
- package/src/loader/layer-weight-shape.js +29 -0
- package/src/loader/load-timing.js +64 -0
- package/src/loader/loader-types.d.ts +3 -0
- package/src/loader/model-executor.js +819 -0
- package/src/loader/model-load-validation.js +15 -0
- package/src/loader/quantization-constants.d.ts +11 -29
- package/src/loader/quantization-constants.js +1 -2
- package/src/loader/shard-resolver.js +0 -14
- package/src/loader/tensor-load-error.js +14 -0
- package/src/loader/tensor-shard-indices.js +8 -0
- package/src/loader/tensors/bf16-loader.js +125 -0
- package/src/loader/tensors/functional-descriptor-loader.js +3 -375
- package/src/loader/tensors/load-executor.js +577 -0
- package/src/loader/tensors/source-transform/executor.js +123 -0
- package/src/loader/tensors/source-transform/plan.js +107 -0
- package/src/loader/tensors/source-transform.js +15 -264
- package/src/loader/tensors/tensor-loader.d.ts +4 -4
- package/src/loader/tensors/tensor-loader.js +7 -653
- package/src/memory/buffer-pool.js +7 -29
- package/src/rules/capability-transform-resolver.js +80 -0
- package/src/rules/execution-contract.d.ts +73 -0
- package/src/rules/execution-contract.js +429 -0
- package/src/rules/execution-rules-contract-check.js +1 -1
- package/src/rules/generated/rule-bundle.json +4305 -0
- package/src/rules/inference/capability-transforms.rules.json +3 -3
- package/src/rules/layer-pattern-contract-check.js +1 -1
- package/src/rules/rule-registry.js +70 -57
- package/src/storage/artifact-storage-context.js +2 -2
- package/src/storage/backend-selection.js +43 -0
- package/src/storage/distribution-transport.js +5 -0
- package/src/storage/download/integrity.d.ts +33 -0
- package/src/storage/download/integrity.js +130 -0
- package/src/storage/download/resume-inspection.d.ts +22 -0
- package/src/storage/download/resume-inspection.js +134 -0
- package/src/storage/download/retry.js +57 -0
- package/src/storage/download/state.js +131 -0
- package/src/storage/download/transport.js +99 -0
- package/src/storage/downloader.js +74 -453
- package/src/storage/model-cache.d.ts +1 -0
- package/src/storage/model-cache.js +51 -11
- package/src/storage/shard-manager.d.ts +5 -0
- package/src/storage/shard-manager.js +50 -369
- package/src/storage/shards/index.js +60 -0
- package/src/storage/shards/integrity.js +225 -0
- package/src/storage/shards/lifecycle.js +22 -0
- package/src/storage/source-artifact-store.js +14 -4
- package/src/tooling/boundary-evidence.d.ts +4 -4
- package/src/tooling/boundary-evidence.js +20 -20
- package/src/tooling/browser-command-runner.js +13 -6
- package/src/tooling/capsule-v2.d.ts +11 -0
- package/src/tooling/capsule-v2.js +58 -0
- package/src/tooling/capsule.js +20 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.js +163 -5
- package/src/tooling/command-api.d.ts +42 -3
- package/src/tooling/command-api.js +7 -2
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
- package/src/tooling/command-runner.html +49 -1
- package/src/tooling/command-validation.js +1 -24
- package/src/tooling/conversion-run-timing.js +24 -0
- package/src/tooling/diagnose-runner.js +11 -9
- package/src/tooling/distribution-shard-transport.js +11 -0
- package/src/tooling/execution-cost-ledger.d.ts +76 -2
- package/src/tooling/execution-cost-ledger.js +197 -15
- package/src/tooling/hf-registry/catalog-io.js +63 -0
- package/src/tooling/litert/package.js +863 -0
- package/src/tooling/litert-package-runtime.js +3 -849
- package/src/tooling/model-capsule-forge.js +478 -0
- package/src/tooling/node-browser/execution.d.ts +40 -0
- package/src/tooling/node-browser/execution.js +308 -0
- package/src/tooling/node-browser/static-server.js +188 -0
- package/src/tooling/node-browser/transport.d.ts +80 -0
- package/src/tooling/node-browser/transport.js +622 -0
- package/src/tooling/node-browser-command-runner.d.ts +18 -78
- package/src/tooling/node-browser-command-runner.js +10 -1112
- package/src/tooling/node-capsule-artifact-store.js +44 -0
- package/src/tooling/node-command-execution.js +237 -0
- package/src/tooling/node-command-runner.js +26 -205
- package/src/tooling/node-converter/input.js +521 -0
- package/src/tooling/node-converter/output.js +244 -0
- package/src/tooling/node-converter-chunk-batches.js +21 -0
- package/src/tooling/node-converter.js +7 -720
- package/src/tooling/node-source-runtime.js +16 -276
- package/src/tooling/node-webgpu-provider.v1.json +33 -0
- package/src/tooling/node-webgpu.js +178 -453
- package/src/tooling/opfs-cache.d.ts +5 -0
- package/src/tooling/opfs-cache.js +4 -0
- package/src/tooling/production-release.d.ts +19 -0
- package/src/tooling/production-release.js +647 -0
- package/src/tooling/program-bundle/embedding-reference.js +19 -0
- package/src/tooling/program-bundle/materialize.js +324 -0
- package/src/tooling/program-bundle/rerank-reference.js +22 -0
- package/src/tooling/program-bundle/sequence-reference.js +42 -0
- package/src/tooling/program-bundle/validation.js +31 -0
- package/src/tooling/program-bundle/wgsl-closure.js +254 -0
- package/src/tooling/program-bundle-parity.d.ts +11 -4
- package/src/tooling/program-bundle-parity.js +262 -109
- package/src/tooling/program-bundle.d.ts +7 -0
- package/src/tooling/program-bundle.js +186 -591
- package/src/tooling/provider-v1-contract.d.ts +61 -0
- package/src/tooling/provider-v1-contract.js +192 -0
- package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
- package/src/tooling/registered-variant-calibration-job.js +210 -0
- package/src/tooling/registered-variant-calibration.d.ts +167 -7
- package/src/tooling/registered-variant-calibration.js +57 -7
- package/src/tooling/runtime-optimization/candidates.js +664 -0
- package/src/tooling/runtime-optimization-campaign.js +166 -0
- package/src/tooling/runtime-optimization-index.js +16 -1
- package/src/tooling/runtime-optimization.d.ts +44 -0
- package/src/tooling/runtime-optimization.js +13 -656
- package/src/tooling/source-intake.d.ts +1 -1
- package/src/tooling/source-intake.js +34 -17
- package/src/tooling/source-package-profiles.js +1 -1
- package/src/tooling/source-runtime/bundle-materialize.js +298 -0
- package/src/tooling/source-runtime/bundle-plan.js +121 -0
- package/src/tooling/source-runtime/node-loader.js +258 -0
- package/src/tooling/source-runtime-bundle.js +26 -411
- package/src/tooling/source-runtime-converter-config.js +1 -1
- package/src/tooling-experimental-exports.js +0 -5
- package/src/tooling-exports/evidence.d.ts +14 -3
- package/src/tooling-exports/evidence.js +14 -3
- package/src/tooling-exports/storage.d.ts +5 -0
- package/src/tooling-exports/storage.js +4 -0
- package/src/tooling-exports.d.ts +3 -0
- package/src/tooling-exports.js +2 -0
- package/src/tooling-exports.shared.d.ts +17 -3
- package/src/tooling-exports.shared.js +9 -3
- package/src/training.d.ts +6 -6
- package/src/training.js +4 -4
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +512 -0
- package/tests/kernels/browser/test-page.js +169 -14
- package/tests/kernels/harness/tolerance.js +1 -0
- package/tests/kernels/reference/index.js +13 -1
- package/tests/kernels/reference/vision.js +160 -0
- package/src/converter/quantizer.d.ts +0 -121
- package/src/gpu/kernels/backward/utils.d.ts +0 -90
- package/src/gpu/kernels/uniform-utils.d.ts +0 -66
- package/src/gpu/kernels/utils.d.ts +0 -147
- package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
- package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
- package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
- package/src/inference/pipelines/text/moe-helpers.js +0 -21
- package/src/rules/converter/execution.rules.json +0 -20
- package/src/rules/converter/tensor-roles.rules.json +0 -13
- package/src/rules/converter/tokenizer.rules.json +0 -12
- package/src/rules/inference/attention.rules.json +0 -69
- package/src/rules/inference/config.rules.json +0 -82
- package/src/rules/inference/dtype.rules.json +0 -113
- package/src/rules/inference/execution.rules.json +0 -195
- package/src/rules/inference/ffn.rules.json +0 -123
- package/src/rules/inference/layer-pattern.rules.json +0 -16
- package/src/rules/inference/layer.rules.json +0 -7
- package/src/rules/inference/moe.rules.json +0 -145
- package/src/rules/kernels/attention.rules.json +0 -68
- package/src/rules/kernels/conv2d.rules.json +0 -6
- package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
- package/src/rules/kernels/dequant.rules.json +0 -58
- package/src/rules/kernels/energy.rules.json +0 -22
- package/src/rules/kernels/fused-ffn.rules.json +0 -15
- package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
- package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
- package/src/rules/kernels/gather.rules.json +0 -93
- package/src/rules/kernels/gelu.rules.json +0 -11
- package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
- package/src/rules/kernels/groupnorm.rules.json +0 -10
- package/src/rules/kernels/kv_quantize.rules.json +0 -9
- package/src/rules/kernels/layernorm.rules.json +0 -6
- package/src/rules/kernels/linear-attention.rules.json +0 -6
- package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
- package/src/rules/kernels/matmul.rules.json +0 -95
- package/src/rules/kernels/modulate.rules.json +0 -6
- package/src/rules/kernels/moe.rules.gptoss.json +0 -105
- package/src/rules/kernels/moe.rules.json +0 -14
- package/src/rules/kernels/moe.rules.mixtral.json +0 -75
- package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
- package/src/rules/kernels/relu.rules.json +0 -6
- package/src/rules/kernels/rep-penalty.rules.json +0 -6
- package/src/rules/kernels/repeat-channels.rules.json +0 -6
- package/src/rules/kernels/residual.rules.json +0 -12
- package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
- package/src/rules/kernels/rmsnorm.rules.json +0 -46
- package/src/rules/kernels/rope-qk.rules.json +0 -6
- package/src/rules/kernels/rope.rules.json +0 -6
- package/src/rules/kernels/sample.rules.json +0 -6
- package/src/rules/kernels/scale.rules.json +0 -8
- package/src/rules/kernels/silu.rules.json +0 -22
- package/src/rules/kernels/softmax.rules.json +0 -25
- package/src/rules/kernels/split-qg.rules.json +0 -6
- package/src/rules/kernels/split-qkv.rules.json +0 -6
- package/src/rules/kernels/upsample2d.rules.json +0 -6
- package/src/rules/loader/tensor-loader.rules.json +0 -42
- package/src/rules/loader/weights.rules.json +0 -42
- package/src/rules/tooling/command-runtime.rules.json +0 -65
- package/src/tooling/hf-registry-utils.js +0 -395
- /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
- /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
- /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
- /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
- /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
- /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
- /package/src/{utils → formats}/array-buffer.js +0 -0
- /package/src/{utils → formats}/canonical-hash.js +0 -0
- /package/src/{utils → formats}/clone-json.js +0 -0
- /package/src/{utils → formats}/encode-utf8.js +0 -0
- /package/src/{utils → formats}/load-json.js +0 -0
- /package/src/{utils → formats}/plain-object.js +0 -0
- /package/src/{utils → formats}/sha256.js +0 -0
- /package/src/{utils → formats}/stable-sort-object.js +0 -0
- /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
- /package/src/{utils → storage}/hf-resolve-url.js +0 -0
- /package/src/{utils → storage}/runtime-env.js +0 -0
- /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
|
@@ -2,6 +2,10 @@ import { log } from '../../../debug/index.js';
|
|
|
2
2
|
import { mergeConfig, dumpConfigSources } from '../../../config/merge.js';
|
|
3
3
|
import { validateModelOverrides } from '../../../config/param-validator.js';
|
|
4
4
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
5
|
+
import { appendHeterogeneousAttentionValidation, resolveHeterogeneousAttentionContract } from './attention/heterogeneous-contract.js';
|
|
6
|
+
import { resolvePostNormContract } from './normalization-contract.js';
|
|
7
|
+
import { resolveEmbeddingNormalization } from './embedding-contract.js';
|
|
8
|
+
import { CHAT_FORMATTER_TYPES } from './chat-format.js';
|
|
5
9
|
import {
|
|
6
10
|
PER_LAYER_INPUT_MATERIALIZATION_MODES,
|
|
7
11
|
PER_LAYER_INPUT_ROW_CACHE_MODES,
|
|
@@ -10,32 +14,14 @@ import {
|
|
|
10
14
|
PER_LAYER_INPUT_HOT_CACHE_MODES,
|
|
11
15
|
PREFILL_CHUNK_SUBMIT_MODES,
|
|
12
16
|
} from '../../../config/schema/execution-v1.schema.js';
|
|
17
|
+
import { assertSupportedManifestInference, validateLayerIntermediateSizesAgainstManifest, validateRequiredInferenceFields } from './config/validation.js';
|
|
18
|
+
import { resolveAudioConfig, resolvePerLayerInputsSession, resolveSessionSettings, resolveVisionConfig } from './config/normalization.js';
|
|
19
|
+
export { assertSupportedManifestInference, validateRequiredInferenceFields } from './config/validation.js';
|
|
13
20
|
|
|
14
21
|
const UNSUPPORTED_RUNTIME_MODEL_TYPES = new Set(['mamba', 'rwkv']);
|
|
15
22
|
|
|
16
|
-
|
|
17
|
-
* Known chat template types that have registered formatters in chat-format.js.
|
|
18
|
-
* Used for load-time validation so unknown types fail early instead of at
|
|
19
|
-
* generation time.
|
|
20
|
-
*/
|
|
21
|
-
const KNOWN_CHAT_TEMPLATE_TYPES = new Set([
|
|
22
|
-
'gemma',
|
|
23
|
-
'gemma4',
|
|
24
|
-
'llama3',
|
|
25
|
-
'gpt-oss',
|
|
26
|
-
'chatml',
|
|
27
|
-
'qwen',
|
|
28
|
-
'translategemma',
|
|
29
|
-
]);
|
|
23
|
+
const KNOWN_CHAT_TEMPLATE_TYPES = new Set(CHAT_FORMATTER_TYPES);
|
|
30
24
|
|
|
31
|
-
/**
|
|
32
|
-
* Validate that a chatTemplate.type value is either null (disabled) or a known
|
|
33
|
-
* formatter type.
|
|
34
|
-
*
|
|
35
|
-
* @param {string | null} type - The chatTemplate.type from manifest inference.
|
|
36
|
-
* @param {string} modelId - Model identifier for diagnostic messages.
|
|
37
|
-
* @returns {boolean} True if the type is valid or null.
|
|
38
|
-
*/
|
|
39
25
|
function validateChatTemplateType(type, modelId) {
|
|
40
26
|
if (type === null || type === undefined) return true;
|
|
41
27
|
if (KNOWN_CHAT_TEMPLATE_TYPES.has(type)) return true;
|
|
@@ -45,330 +31,6 @@ function validateChatTemplateType(type, modelId) {
|
|
|
45
31
|
);
|
|
46
32
|
}
|
|
47
33
|
|
|
48
|
-
function resolvePerLayerInputsSession(inferenceConfig, modelId) {
|
|
49
|
-
const sessionConfig = inferenceConfig?.session?.perLayerInputs;
|
|
50
|
-
if (sessionConfig === undefined) {
|
|
51
|
-
throw new Error(
|
|
52
|
-
`Manifest "${modelId}" is missing inference.session.perLayerInputs. ` +
|
|
53
|
-
'Re-convert the model so per-layer input materialization policy is explicit.'
|
|
54
|
-
);
|
|
55
|
-
}
|
|
56
|
-
if (!sessionConfig || typeof sessionConfig !== 'object' || Array.isArray(sessionConfig)) {
|
|
57
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.perLayerInputs.`);
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
const materialization = sessionConfig.materialization;
|
|
61
|
-
if (!PER_LAYER_INPUT_MATERIALIZATION_MODES.includes(materialization)) {
|
|
62
|
-
throw new Error(
|
|
63
|
-
`Manifest "${modelId}" has invalid inference.session.perLayerInputs.materialization ` +
|
|
64
|
-
`"${String(materialization)}".`
|
|
65
|
-
);
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
const rowCache = sessionConfig.rowCache;
|
|
69
|
-
if (!rowCache || typeof rowCache !== 'object' || Array.isArray(rowCache)) {
|
|
70
|
-
throw new Error(`Manifest "${modelId}" is missing inference.session.perLayerInputs.rowCache.`);
|
|
71
|
-
}
|
|
72
|
-
if (!PER_LAYER_INPUT_ROW_CACHE_MODES.includes(rowCache.mode)) {
|
|
73
|
-
throw new Error(
|
|
74
|
-
`Manifest "${modelId}" has invalid inference.session.perLayerInputs.rowCache.mode ` +
|
|
75
|
-
`"${String(rowCache.mode)}".`
|
|
76
|
-
);
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
const maxRows = Math.trunc(Number(rowCache.maxRows));
|
|
80
|
-
if (!Number.isFinite(maxRows) || maxRows <= 0) {
|
|
81
|
-
throw new Error(
|
|
82
|
-
`Manifest "${modelId}" requires inference.session.perLayerInputs.rowCache.maxRows ` +
|
|
83
|
-
`to be a positive integer; got ${String(rowCache.maxRows)}.`
|
|
84
|
-
);
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
const maxBytes = Math.trunc(Number(rowCache.maxBytes));
|
|
88
|
-
if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
|
|
89
|
-
throw new Error(
|
|
90
|
-
`Manifest "${modelId}" requires inference.session.perLayerInputs.rowCache.maxBytes ` +
|
|
91
|
-
`to be a positive integer; got ${String(rowCache.maxBytes)}.`
|
|
92
|
-
);
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
const decodedDtype = String(rowCache.decodedDtype ?? '').toLowerCase();
|
|
96
|
-
if (decodedDtype !== 'f16' && decodedDtype !== 'f32') {
|
|
97
|
-
throw new Error(
|
|
98
|
-
`Manifest "${modelId}" has invalid inference.session.perLayerInputs.rowCache.decodedDtype ` +
|
|
99
|
-
`"${String(rowCache.decodedDtype)}".`
|
|
100
|
-
);
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
const prefetch = sessionConfig.prefetch;
|
|
104
|
-
if (!prefetch || typeof prefetch !== 'object' || Array.isArray(prefetch)) {
|
|
105
|
-
throw new Error(`Manifest "${modelId}" is missing inference.session.perLayerInputs.prefetch.`);
|
|
106
|
-
}
|
|
107
|
-
if (!PER_LAYER_INPUT_PREFETCH_MODES.includes(prefetch.mode)) {
|
|
108
|
-
throw new Error(
|
|
109
|
-
`Manifest "${modelId}" has invalid inference.session.perLayerInputs.prefetch.mode ` +
|
|
110
|
-
`"${String(prefetch.mode)}".`
|
|
111
|
-
);
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
const rowsAhead = Math.trunc(Number(prefetch.rowsAhead));
|
|
115
|
-
if (!Number.isFinite(rowsAhead) || rowsAhead <= 0) {
|
|
116
|
-
throw new Error(
|
|
117
|
-
`Manifest "${modelId}" requires inference.session.perLayerInputs.prefetch.rowsAhead ` +
|
|
118
|
-
`to be a positive integer; got ${String(prefetch.rowsAhead)}.`
|
|
119
|
-
);
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
const gpuUpload = sessionConfig.gpuUpload;
|
|
123
|
-
if (!gpuUpload || typeof gpuUpload !== 'object' || Array.isArray(gpuUpload)) {
|
|
124
|
-
throw new Error(`Manifest "${modelId}" is missing inference.session.perLayerInputs.gpuUpload.`);
|
|
125
|
-
}
|
|
126
|
-
if (!PER_LAYER_INPUT_GPU_UPLOAD_MODES.includes(gpuUpload.mode)) {
|
|
127
|
-
throw new Error(
|
|
128
|
-
`Manifest "${modelId}" has invalid inference.session.perLayerInputs.gpuUpload.mode ` +
|
|
129
|
-
`"${String(gpuUpload.mode)}".`
|
|
130
|
-
);
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
const stagingRows = Math.trunc(Number(gpuUpload.stagingRows));
|
|
134
|
-
if (!Number.isFinite(stagingRows) || stagingRows <= 0) {
|
|
135
|
-
throw new Error(
|
|
136
|
-
`Manifest "${modelId}" requires inference.session.perLayerInputs.gpuUpload.stagingRows ` +
|
|
137
|
-
`to be a positive integer; got ${String(gpuUpload.stagingRows)}.`
|
|
138
|
-
);
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
const hotCache = sessionConfig.hotCache;
|
|
142
|
-
if (!hotCache || typeof hotCache !== 'object' || Array.isArray(hotCache)) {
|
|
143
|
-
throw new Error(`Manifest "${modelId}" is missing inference.session.perLayerInputs.hotCache.`);
|
|
144
|
-
}
|
|
145
|
-
if (!PER_LAYER_INPUT_HOT_CACHE_MODES.includes(hotCache.mode)) {
|
|
146
|
-
throw new Error(
|
|
147
|
-
`Manifest "${modelId}" has invalid inference.session.perLayerInputs.hotCache.mode ` +
|
|
148
|
-
`"${String(hotCache.mode)}".`
|
|
149
|
-
);
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
const hotMaxTokens = Math.trunc(Number(hotCache.maxTokens));
|
|
153
|
-
if (!Number.isFinite(hotMaxTokens) || hotMaxTokens <= 0) {
|
|
154
|
-
throw new Error(
|
|
155
|
-
`Manifest "${modelId}" requires inference.session.perLayerInputs.hotCache.maxTokens ` +
|
|
156
|
-
`to be a positive integer; got ${String(hotCache.maxTokens)}.`
|
|
157
|
-
);
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
const hotMaxBytes = Math.trunc(Number(hotCache.maxBytes));
|
|
161
|
-
if (!Number.isFinite(hotMaxBytes) || hotMaxBytes <= 0) {
|
|
162
|
-
throw new Error(
|
|
163
|
-
`Manifest "${modelId}" requires inference.session.perLayerInputs.hotCache.maxBytes ` +
|
|
164
|
-
`to be a positive integer; got ${String(hotCache.maxBytes)}.`
|
|
165
|
-
);
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
const hotOutputDtype = String(hotCache.outputDtype ?? '').toLowerCase();
|
|
169
|
-
if (hotOutputDtype !== 'f16' && hotOutputDtype !== 'f32') {
|
|
170
|
-
throw new Error(
|
|
171
|
-
`Manifest "${modelId}" has invalid inference.session.perLayerInputs.hotCache.outputDtype ` +
|
|
172
|
-
`"${String(hotCache.outputDtype)}".`
|
|
173
|
-
);
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
return {
|
|
177
|
-
materialization,
|
|
178
|
-
rowCache: {
|
|
179
|
-
mode: rowCache.mode,
|
|
180
|
-
maxRows,
|
|
181
|
-
maxBytes,
|
|
182
|
-
decodedDtype,
|
|
183
|
-
},
|
|
184
|
-
prefetch: {
|
|
185
|
-
mode: prefetch.mode,
|
|
186
|
-
rowsAhead,
|
|
187
|
-
},
|
|
188
|
-
gpuUpload: {
|
|
189
|
-
mode: gpuUpload.mode,
|
|
190
|
-
stagingRows,
|
|
191
|
-
},
|
|
192
|
-
hotCache: {
|
|
193
|
-
mode: hotCache.mode,
|
|
194
|
-
maxTokens: hotMaxTokens,
|
|
195
|
-
maxBytes: hotMaxBytes,
|
|
196
|
-
outputDtype: hotOutputDtype,
|
|
197
|
-
},
|
|
198
|
-
};
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
function resolveSessionSettings(inferenceConfig, modelId) {
|
|
202
|
-
// All four fields are optional on the manifest today for backwards compatibility;
|
|
203
|
-
// when absent we fall through to the runtime config (getRuntimeConfig()). When
|
|
204
|
-
// present on the manifest, manifest wins unless an explicit runtime profile overrides.
|
|
205
|
-
// The merge layer in src/config/merge.js already applies that precedence — this
|
|
206
|
-
// resolver just validates the manifest-supplied values and normalizes them.
|
|
207
|
-
const session = inferenceConfig?.session;
|
|
208
|
-
const submit = session?.prefillChunkSubmitMode;
|
|
209
|
-
if (submit !== undefined && submit !== null && !PREFILL_CHUNK_SUBMIT_MODES.includes(submit)) {
|
|
210
|
-
throw new Error(
|
|
211
|
-
`Manifest "${modelId}" has invalid inference.session.prefillChunkSubmitMode ` +
|
|
212
|
-
`"${String(submit)}"; expected one of ${PREFILL_CHUNK_SUBMIT_MODES.join(', ')}.`
|
|
213
|
-
);
|
|
214
|
-
}
|
|
215
|
-
const tokenChunk = session?.prefillTokenChunkSize;
|
|
216
|
-
if (
|
|
217
|
-
tokenChunk !== undefined
|
|
218
|
-
&& tokenChunk !== null
|
|
219
|
-
&& (!Number.isInteger(tokenChunk) || tokenChunk <= 0)
|
|
220
|
-
) {
|
|
221
|
-
throw new Error(
|
|
222
|
-
`Manifest "${modelId}" has invalid inference.session.prefillTokenChunkSize ` +
|
|
223
|
-
`"${String(tokenChunk)}"; expected null or a positive integer.`
|
|
224
|
-
);
|
|
225
|
-
}
|
|
226
|
-
const skipEmbeddingKVCacheWrites = session?.skipEmbeddingKVCacheWrites;
|
|
227
|
-
if (
|
|
228
|
-
skipEmbeddingKVCacheWrites !== undefined
|
|
229
|
-
&& skipEmbeddingKVCacheWrites !== null
|
|
230
|
-
&& typeof skipEmbeddingKVCacheWrites !== 'boolean'
|
|
231
|
-
) {
|
|
232
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.skipEmbeddingKVCacheWrites "${String(skipEmbeddingKVCacheWrites)}"; expected boolean.`);
|
|
233
|
-
}
|
|
234
|
-
const flash = session?.useFlashPrefillAttention;
|
|
235
|
-
if (flash !== undefined && flash !== null && typeof flash !== 'boolean') {
|
|
236
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFlashPrefillAttention "${String(flash)}"; expected boolean.`);
|
|
237
|
-
}
|
|
238
|
-
const largeBatchF16F32FusedGateUp = session?.useLargeBatchF16F32FusedGateUp;
|
|
239
|
-
if (largeBatchF16F32FusedGateUp !== undefined && largeBatchF16F32FusedGateUp !== null && typeof largeBatchF16F32FusedGateUp !== 'boolean') {
|
|
240
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.useLargeBatchF16F32FusedGateUp "${String(largeBatchF16F32FusedGateUp)}"; expected boolean.`);
|
|
241
|
-
}
|
|
242
|
-
const wide = session?.useWideTileQ4KPrefill;
|
|
243
|
-
if (wide !== undefined && wide !== null && typeof wide !== 'boolean') {
|
|
244
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileQ4KPrefill "${String(wide)}"; expected boolean.`);
|
|
245
|
-
}
|
|
246
|
-
const wideDecode = session?.useWideTileQ4KDecode;
|
|
247
|
-
if (wideDecode !== undefined && wideDecode !== null && typeof wideDecode !== 'boolean') {
|
|
248
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileQ4KDecode "${String(wideDecode)}"; expected boolean.`);
|
|
249
|
-
}
|
|
250
|
-
const sandwichRmsNormPair = session?.useSandwichRMSNormPairFusion;
|
|
251
|
-
if (sandwichRmsNormPair !== undefined && sandwichRmsNormPair !== null && typeof sandwichRmsNormPair !== 'boolean') {
|
|
252
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.useSandwichRMSNormPairFusion "${String(sandwichRmsNormPair)}"; expected boolean.`);
|
|
253
|
-
}
|
|
254
|
-
const postFfnNextInputRmsNormPair = session?.usePostFfnNextInputRMSNormPairFusion;
|
|
255
|
-
if (postFfnNextInputRmsNormPair !== undefined && postFfnNextInputRmsNormPair !== null && typeof postFfnNextInputRmsNormPair !== 'boolean') {
|
|
256
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.usePostFfnNextInputRMSNormPairFusion "${String(postFfnNextInputRmsNormPair)}"; expected boolean.`);
|
|
257
|
-
}
|
|
258
|
-
const postAttnNormFusedGateUp = session?.usePostAttnNormFusedGateUp;
|
|
259
|
-
if (postAttnNormFusedGateUp !== undefined && postAttnNormFusedGateUp !== null && typeof postAttnNormFusedGateUp !== 'boolean') {
|
|
260
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.usePostAttnNormFusedGateUp "${String(postAttnNormFusedGateUp)}"; expected boolean.`);
|
|
261
|
-
}
|
|
262
|
-
const linearAttentionABProjectionFusion = session?.useLinearAttentionABProjectionFusion;
|
|
263
|
-
if (linearAttentionABProjectionFusion !== undefined && linearAttentionABProjectionFusion !== null && typeof linearAttentionABProjectionFusion !== 'boolean') {
|
|
264
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.useLinearAttentionABProjectionFusion "${String(linearAttentionABProjectionFusion)}"; expected boolean.`);
|
|
265
|
-
}
|
|
266
|
-
const linearAttentionQKVZProjectionFusion = session?.useLinearAttentionQKVZProjectionFusion;
|
|
267
|
-
if (linearAttentionQKVZProjectionFusion !== undefined && linearAttentionQKVZProjectionFusion !== null && typeof linearAttentionQKVZProjectionFusion !== 'boolean') {
|
|
268
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.useLinearAttentionQKVZProjectionFusion "${String(linearAttentionQKVZProjectionFusion)}"; expected boolean.`);
|
|
269
|
-
}
|
|
270
|
-
const linearAttentionFusedDecodeCore = session?.useLinearAttentionFusedDecodeCore;
|
|
271
|
-
if (linearAttentionFusedDecodeCore !== undefined && linearAttentionFusedDecodeCore !== null && typeof linearAttentionFusedDecodeCore !== 'boolean') {
|
|
272
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.useLinearAttentionFusedDecodeCore "${String(linearAttentionFusedDecodeCore)}"; expected boolean.`);
|
|
273
|
-
}
|
|
274
|
-
const wideTileResidualFusion = session?.useWideTileResidualFusion;
|
|
275
|
-
if (wideTileResidualFusion !== undefined && wideTileResidualFusion !== null && typeof wideTileResidualFusion !== 'boolean') {
|
|
276
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileResidualFusion "${String(wideTileResidualFusion)}"; expected boolean.`);
|
|
277
|
-
}
|
|
278
|
-
const fusedRmsnormWideTile = session?.useFusedRmsnormWideTile;
|
|
279
|
-
if (fusedRmsnormWideTile !== undefined && fusedRmsnormWideTile !== null && typeof fusedRmsnormWideTile !== 'boolean') {
|
|
280
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFusedRmsnormWideTile "${String(fusedRmsnormWideTile)}"; expected boolean.`);
|
|
281
|
-
}
|
|
282
|
-
const fusedQKVSplitQKNorm = session?.useFusedQKVSplitQKNorm;
|
|
283
|
-
if (fusedQKVSplitQKNorm !== undefined && fusedQKVSplitQKNorm !== null && typeof fusedQKVSplitQKNorm !== 'boolean') {
|
|
284
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFusedQKVSplitQKNorm "${String(fusedQKVSplitQKNorm)}"; expected boolean.`);
|
|
285
|
-
}
|
|
286
|
-
const fusedQKVSplitQKNormRoPE = session?.useFusedQKVSplitQKNormRoPE;
|
|
287
|
-
if (fusedQKVSplitQKNormRoPE !== undefined && fusedQKVSplitQKNormRoPE !== null && typeof fusedQKVSplitQKNormRoPE !== 'boolean') {
|
|
288
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFusedQKVSplitQKNormRoPE "${String(fusedQKVSplitQKNormRoPE)}"; expected boolean.`);
|
|
289
|
-
}
|
|
290
|
-
const retain = session?.retainQ4KMaterialization;
|
|
291
|
-
if (retain !== undefined && retain !== null && typeof retain !== 'boolean') {
|
|
292
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.retainQ4KMaterialization "${String(retain)}"; expected boolean.`);
|
|
293
|
-
}
|
|
294
|
-
const lmHeadArgmaxQ4K = session?.lmHeadArgmaxQ4K;
|
|
295
|
-
if (lmHeadArgmaxQ4K !== undefined && lmHeadArgmaxQ4K !== null) {
|
|
296
|
-
if (typeof lmHeadArgmaxQ4K !== 'object' || Array.isArray(lmHeadArgmaxQ4K)) {
|
|
297
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K; expected object or null.`);
|
|
298
|
-
}
|
|
299
|
-
const fullBlockFastPath = lmHeadArgmaxQ4K.useFullBlockFastPath;
|
|
300
|
-
if (fullBlockFastPath !== undefined && typeof fullBlockFastPath !== 'boolean') {
|
|
301
|
-
throw new Error(
|
|
302
|
-
`Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K.useFullBlockFastPath ` +
|
|
303
|
-
`"${String(fullBlockFastPath)}"; expected boolean.`
|
|
304
|
-
);
|
|
305
|
-
}
|
|
306
|
-
const colsPerWorkgroup = lmHeadArgmaxQ4K.colsPerWorkgroup;
|
|
307
|
-
if (colsPerWorkgroup !== undefined && (!Number.isInteger(colsPerWorkgroup) || colsPerWorkgroup <= 0)) {
|
|
308
|
-
throw new Error(
|
|
309
|
-
`Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K.colsPerWorkgroup ` +
|
|
310
|
-
`"${String(colsPerWorkgroup)}"; expected positive integer.`
|
|
311
|
-
);
|
|
312
|
-
}
|
|
313
|
-
const threadsPerCol = lmHeadArgmaxQ4K.threadsPerCol;
|
|
314
|
-
if (threadsPerCol !== undefined && (!Number.isInteger(threadsPerCol) || threadsPerCol <= 0)) {
|
|
315
|
-
throw new Error(
|
|
316
|
-
`Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K.threadsPerCol ` +
|
|
317
|
-
`"${String(threadsPerCol)}"; expected positive integer.`
|
|
318
|
-
);
|
|
319
|
-
}
|
|
320
|
-
}
|
|
321
|
-
const attentionDecodeOnline = session?.attentionDecodeOnline;
|
|
322
|
-
if (attentionDecodeOnline !== undefined && attentionDecodeOnline !== null) {
|
|
323
|
-
if (typeof attentionDecodeOnline !== 'object' || Array.isArray(attentionDecodeOnline)) {
|
|
324
|
-
throw new Error(`Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline; expected object or null.`);
|
|
325
|
-
}
|
|
326
|
-
const workgroupSize = attentionDecodeOnline.workgroupSize;
|
|
327
|
-
if (workgroupSize !== undefined && workgroupSize !== 128 && workgroupSize !== 256) {
|
|
328
|
-
throw new Error(
|
|
329
|
-
`Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline.workgroupSize ` +
|
|
330
|
-
`"${String(workgroupSize)}"; expected 128 or 256.`
|
|
331
|
-
);
|
|
332
|
-
}
|
|
333
|
-
const directKv = attentionDecodeOnline.useDirectContiguousKVLayout;
|
|
334
|
-
if (directKv !== undefined && typeof directKv !== 'boolean') {
|
|
335
|
-
throw new Error(
|
|
336
|
-
`Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline.useDirectContiguousKVLayout ` +
|
|
337
|
-
`"${String(directKv)}"; expected boolean.`
|
|
338
|
-
);
|
|
339
|
-
}
|
|
340
|
-
const outputGateFusion = attentionDecodeOnline.useOutputGateFusion;
|
|
341
|
-
if (outputGateFusion !== undefined && typeof outputGateFusion !== 'boolean') {
|
|
342
|
-
throw new Error(
|
|
343
|
-
`Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline.useOutputGateFusion ` +
|
|
344
|
-
`"${String(outputGateFusion)}"; expected boolean.`
|
|
345
|
-
);
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
return {
|
|
349
|
-
prefillChunkSubmitMode: submit ?? null,
|
|
350
|
-
prefillTokenChunkSize: tokenChunk ?? null,
|
|
351
|
-
skipEmbeddingKVCacheWrites: skipEmbeddingKVCacheWrites ?? null,
|
|
352
|
-
useFlashPrefillAttention: flash ?? null,
|
|
353
|
-
useLargeBatchF16F32FusedGateUp: largeBatchF16F32FusedGateUp ?? null,
|
|
354
|
-
useWideTileQ4KPrefill: wide ?? null,
|
|
355
|
-
useWideTileQ4KDecode: wideDecode ?? null,
|
|
356
|
-
useSandwichRMSNormPairFusion: sandwichRmsNormPair ?? null,
|
|
357
|
-
usePostFfnNextInputRMSNormPairFusion: postFfnNextInputRmsNormPair ?? null,
|
|
358
|
-
usePostAttnNormFusedGateUp: postAttnNormFusedGateUp ?? null,
|
|
359
|
-
useLinearAttentionABProjectionFusion: linearAttentionABProjectionFusion ?? null,
|
|
360
|
-
useLinearAttentionQKVZProjectionFusion: linearAttentionQKVZProjectionFusion ?? null,
|
|
361
|
-
useLinearAttentionFusedDecodeCore: linearAttentionFusedDecodeCore ?? null,
|
|
362
|
-
useWideTileResidualFusion: wideTileResidualFusion ?? null,
|
|
363
|
-
useFusedRmsnormWideTile: fusedRmsnormWideTile ?? null,
|
|
364
|
-
useFusedQKVSplitQKNorm: fusedQKVSplitQKNorm ?? null,
|
|
365
|
-
useFusedQKVSplitQKNormRoPE: fusedQKVSplitQKNormRoPE ?? null,
|
|
366
|
-
retainQ4KMaterialization: retain ?? null,
|
|
367
|
-
lmHeadArgmaxQ4K: lmHeadArgmaxQ4K ?? null,
|
|
368
|
-
attentionDecodeOnline: attentionDecodeOnline ?? null,
|
|
369
|
-
};
|
|
370
|
-
}
|
|
371
|
-
|
|
372
34
|
function resolveLargeWeightsConfig(inferenceConfig, modelId) {
|
|
373
35
|
const lw = inferenceConfig?.largeWeights;
|
|
374
36
|
if (lw === undefined || lw === null) return { gpuResidentOverrides: null };
|
|
@@ -471,15 +133,6 @@ export function getStopTokenIds(manifest) {
|
|
|
471
133
|
);
|
|
472
134
|
}
|
|
473
135
|
|
|
474
|
-
function normalizeFfnTensorShape(value) {
|
|
475
|
-
if (!Array.isArray(value) || value.length !== 2) return null;
|
|
476
|
-
const rows = Number(value[0]);
|
|
477
|
-
const cols = Number(value[1]);
|
|
478
|
-
if (!Number.isFinite(rows) || !Number.isFinite(cols)) return null;
|
|
479
|
-
if (rows <= 0 || cols <= 0) return null;
|
|
480
|
-
return [Math.trunc(rows), Math.trunc(cols)];
|
|
481
|
-
}
|
|
482
|
-
|
|
483
136
|
function resolveIntermediateSizeForRuntime(arch) {
|
|
484
137
|
return arch?.intermediateSize;
|
|
485
138
|
}
|
|
@@ -533,113 +186,6 @@ function buildPerLayerIntermediateSizes({
|
|
|
533
186
|
return intermediateSizes;
|
|
534
187
|
}
|
|
535
188
|
|
|
536
|
-
function getDenseFfnTensorShape(tensors, names) {
|
|
537
|
-
if (!tensors || typeof tensors !== 'object') return null;
|
|
538
|
-
for (const name of names) {
|
|
539
|
-
const shape = normalizeFfnTensorShape(tensors[name]?.shape);
|
|
540
|
-
if (shape) return shape;
|
|
541
|
-
}
|
|
542
|
-
return null;
|
|
543
|
-
}
|
|
544
|
-
|
|
545
|
-
function assertDenseFfnTensorShape(modelId, layerIdx, label, actualShape, expectedShape) {
|
|
546
|
-
if (!actualShape) return;
|
|
547
|
-
if (actualShape[0] === expectedShape[0] && actualShape[1] === expectedShape[1]) {
|
|
548
|
-
return;
|
|
549
|
-
}
|
|
550
|
-
throw new Error(
|
|
551
|
-
`Manifest "${modelId}" layer ${layerIdx} ${label} shape [${actualShape.join(', ')}] does not match ` +
|
|
552
|
-
`the resolved FFN contract [${expectedShape.join(', ')}]. Re-convert the model so manifest inference ` +
|
|
553
|
-
'and FFN tensor shapes agree.'
|
|
554
|
-
);
|
|
555
|
-
}
|
|
556
|
-
|
|
557
|
-
function validateLayerIntermediateSizesAgainstManifest(manifest, hiddenSize, layerIntermediateSizes, modelId) {
|
|
558
|
-
const tensors = manifest?.tensors;
|
|
559
|
-
if (!tensors || typeof tensors !== 'object' || !Array.isArray(layerIntermediateSizes)) {
|
|
560
|
-
return;
|
|
561
|
-
}
|
|
562
|
-
|
|
563
|
-
for (let layerIdx = 0; layerIdx < layerIntermediateSizes.length; layerIdx += 1) {
|
|
564
|
-
const intermediateSize = Number(layerIntermediateSizes[layerIdx]);
|
|
565
|
-
if (!Number.isFinite(intermediateSize) || intermediateSize <= 0) {
|
|
566
|
-
throw new Error(
|
|
567
|
-
`Manifest "${modelId}" resolved an invalid FFN intermediate size (${String(layerIntermediateSizes[layerIdx])}) ` +
|
|
568
|
-
`for layer ${layerIdx}.`
|
|
569
|
-
);
|
|
570
|
-
}
|
|
571
|
-
|
|
572
|
-
const expectedIntermediateSize = Math.trunc(intermediateSize);
|
|
573
|
-
const languagePrefix = `model.language_model.layers.${layerIdx}`;
|
|
574
|
-
const genericPrefix = `model.layers.${layerIdx}`;
|
|
575
|
-
const gateShape = getDenseFfnTensorShape(tensors, [
|
|
576
|
-
`${languagePrefix}.mlp.gate_proj.weight`,
|
|
577
|
-
`${genericPrefix}.mlp.gate_proj.weight`,
|
|
578
|
-
`${languagePrefix}.ffn.gate_proj.weight`,
|
|
579
|
-
`${languagePrefix}.ffn_gate.weight`,
|
|
580
|
-
`${languagePrefix}.feed_forward.w1.weight`,
|
|
581
|
-
`${genericPrefix}.feed_forward.w1.weight`,
|
|
582
|
-
`layers.${layerIdx}.feed_forward.w1.weight`,
|
|
583
|
-
]);
|
|
584
|
-
const upShape = getDenseFfnTensorShape(tensors, [
|
|
585
|
-
`${languagePrefix}.mlp.up_proj.weight`,
|
|
586
|
-
`${genericPrefix}.mlp.up_proj.weight`,
|
|
587
|
-
`${languagePrefix}.ffn.up_proj.weight`,
|
|
588
|
-
`${languagePrefix}.ffn_up.weight`,
|
|
589
|
-
`${languagePrefix}.feed_forward.w3.weight`,
|
|
590
|
-
`${genericPrefix}.feed_forward.w3.weight`,
|
|
591
|
-
`layers.${layerIdx}.feed_forward.w3.weight`,
|
|
592
|
-
]);
|
|
593
|
-
const downShape = getDenseFfnTensorShape(tensors, [
|
|
594
|
-
`${languagePrefix}.mlp.down_proj.weight`,
|
|
595
|
-
`${genericPrefix}.mlp.down_proj.weight`,
|
|
596
|
-
`${languagePrefix}.ffn.down_proj.weight`,
|
|
597
|
-
`${languagePrefix}.ffn_down.weight`,
|
|
598
|
-
`${languagePrefix}.feed_forward.w2.weight`,
|
|
599
|
-
`${genericPrefix}.feed_forward.w2.weight`,
|
|
600
|
-
`layers.${layerIdx}.feed_forward.w2.weight`,
|
|
601
|
-
]);
|
|
602
|
-
const gateUpShape = getDenseFfnTensorShape(tensors, [
|
|
603
|
-
`${languagePrefix}.mlp.gate_up_proj.weight`,
|
|
604
|
-
`${genericPrefix}.mlp.gate_up_proj.weight`,
|
|
605
|
-
`${languagePrefix}.ffn.gate_up_proj.weight`,
|
|
606
|
-
`${languagePrefix}.ffn_gate_up.weight`,
|
|
607
|
-
`${languagePrefix}.feed_forward.w1_w3.weight`,
|
|
608
|
-
`${genericPrefix}.feed_forward.w1_w3.weight`,
|
|
609
|
-
`layers.${layerIdx}.feed_forward.w1_w3.weight`,
|
|
610
|
-
]);
|
|
611
|
-
|
|
612
|
-
assertDenseFfnTensorShape(
|
|
613
|
-
modelId,
|
|
614
|
-
layerIdx,
|
|
615
|
-
'gate weight',
|
|
616
|
-
gateShape,
|
|
617
|
-
[expectedIntermediateSize, hiddenSize]
|
|
618
|
-
);
|
|
619
|
-
assertDenseFfnTensorShape(
|
|
620
|
-
modelId,
|
|
621
|
-
layerIdx,
|
|
622
|
-
'up weight',
|
|
623
|
-
upShape,
|
|
624
|
-
[expectedIntermediateSize, hiddenSize]
|
|
625
|
-
);
|
|
626
|
-
assertDenseFfnTensorShape(
|
|
627
|
-
modelId,
|
|
628
|
-
layerIdx,
|
|
629
|
-
'down weight',
|
|
630
|
-
downShape,
|
|
631
|
-
[hiddenSize, expectedIntermediateSize]
|
|
632
|
-
);
|
|
633
|
-
assertDenseFfnTensorShape(
|
|
634
|
-
modelId,
|
|
635
|
-
layerIdx,
|
|
636
|
-
'gate_up weight',
|
|
637
|
-
gateUpShape,
|
|
638
|
-
[expectedIntermediateSize * 2, hiddenSize]
|
|
639
|
-
);
|
|
640
|
-
}
|
|
641
|
-
}
|
|
642
|
-
|
|
643
189
|
export function resolveLayerIntermediateSize(config, layerIdx) {
|
|
644
190
|
const intermediateSizes = Array.isArray(config?.intermediateSizes) ? config.intermediateSizes : null;
|
|
645
191
|
if (Number.isFinite(layerIdx) && intermediateSizes) {
|
|
@@ -660,470 +206,10 @@ export function resolveLayerIntermediateSize(config, layerIdx) {
|
|
|
660
206
|
// Manifest-First Config Resolution (NEW)
|
|
661
207
|
// =============================================================================
|
|
662
208
|
|
|
663
|
-
|
|
664
209
|
function hasManifestInference(manifest) {
|
|
665
210
|
return 'inference' in manifest && manifest.inference != null;
|
|
666
211
|
}
|
|
667
212
|
|
|
668
|
-
function normalizeUnsupportedText(value) {
|
|
669
|
-
if (typeof value !== 'string') return '';
|
|
670
|
-
return value.trim();
|
|
671
|
-
}
|
|
672
|
-
|
|
673
|
-
export function assertSupportedManifestInference(manifest) {
|
|
674
|
-
const modelId = manifest?.modelId ?? 'unknown';
|
|
675
|
-
const unsupported = manifest?.inference?.unsupported;
|
|
676
|
-
if (unsupported == null) {
|
|
677
|
-
return;
|
|
678
|
-
}
|
|
679
|
-
if (typeof unsupported !== 'object' || Array.isArray(unsupported)) {
|
|
680
|
-
throw new Error(
|
|
681
|
-
`Manifest "${modelId}" has invalid inference.unsupported. ` +
|
|
682
|
-
'Expected null or an object with code, message, and recommendation fields.'
|
|
683
|
-
);
|
|
684
|
-
}
|
|
685
|
-
const code = normalizeUnsupportedText(unsupported.code) || 'unsupported-manifest-contract';
|
|
686
|
-
const message = normalizeUnsupportedText(unsupported.message);
|
|
687
|
-
const recommendation = normalizeUnsupportedText(unsupported.recommendation);
|
|
688
|
-
throw new Error(
|
|
689
|
-
`Manifest "${modelId}" is not supported by Doppler runtime (${code}).` +
|
|
690
|
-
(message ? ` ${message}` : '') +
|
|
691
|
-
(recommendation ? ` ${recommendation}` : '')
|
|
692
|
-
);
|
|
693
|
-
}
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
export function validateRequiredInferenceFields(inf, modelId) {
|
|
697
|
-
inf = inf ?? {};
|
|
698
|
-
inf.attention = inf.attention ?? {};
|
|
699
|
-
inf.normalization = inf.normalization ?? {};
|
|
700
|
-
inf.ffn = inf.ffn ?? {};
|
|
701
|
-
inf.rope = inf.rope ?? {};
|
|
702
|
-
inf.output = inf.output ?? {};
|
|
703
|
-
inf.layerPattern = inf.layerPattern ?? {};
|
|
704
|
-
inf.chatTemplate = inf.chatTemplate ?? {};
|
|
705
|
-
const errors = [];
|
|
706
|
-
|
|
707
|
-
// Attention fields - non-nullable required
|
|
708
|
-
if (inf.attention.queryPreAttnScalar == null) {
|
|
709
|
-
errors.push('attention.queryPreAttnScalar is required');
|
|
710
|
-
}
|
|
711
|
-
if (inf.attention.queryKeyNorm == null) {
|
|
712
|
-
errors.push('attention.queryKeyNorm is required');
|
|
713
|
-
}
|
|
714
|
-
if (
|
|
715
|
-
inf.attention.queryKeyNormType !== undefined
|
|
716
|
-
&& !['rmsnorm', 'layernorm'].includes(inf.attention.queryKeyNormType)
|
|
717
|
-
) {
|
|
718
|
-
errors.push('attention.queryKeyNormType must be "rmsnorm" or "layernorm"');
|
|
719
|
-
}
|
|
720
|
-
if (
|
|
721
|
-
inf.attention.queryKeyNormAxis !== undefined
|
|
722
|
-
&& !['head', 'projection'].includes(inf.attention.queryKeyNormAxis)
|
|
723
|
-
) {
|
|
724
|
-
errors.push('attention.queryKeyNormAxis must be "head" or "projection"');
|
|
725
|
-
}
|
|
726
|
-
if (
|
|
727
|
-
inf.attention.queryKeyNormLayers !== undefined
|
|
728
|
-
&& inf.attention.queryKeyNormLayers !== null
|
|
729
|
-
) {
|
|
730
|
-
if (!Array.isArray(inf.attention.queryKeyNormLayers)) {
|
|
731
|
-
errors.push('attention.queryKeyNormLayers must be null or an array of layer indices');
|
|
732
|
-
} else {
|
|
733
|
-
for (const layerIdx of inf.attention.queryKeyNormLayers) {
|
|
734
|
-
if (!Number.isInteger(layerIdx) || layerIdx < 0) {
|
|
735
|
-
errors.push('attention.queryKeyNormLayers must contain only non-negative integer layer indices');
|
|
736
|
-
break;
|
|
737
|
-
}
|
|
738
|
-
}
|
|
739
|
-
}
|
|
740
|
-
}
|
|
741
|
-
if (
|
|
742
|
-
inf.attention.queryKeyNormWeightLayers !== undefined
|
|
743
|
-
&& inf.attention.queryKeyNormWeightLayers !== null
|
|
744
|
-
) {
|
|
745
|
-
if (!Array.isArray(inf.attention.queryKeyNormWeightLayers)) {
|
|
746
|
-
errors.push('attention.queryKeyNormWeightLayers must be null or an array of layer indices');
|
|
747
|
-
} else {
|
|
748
|
-
for (const layerIdx of inf.attention.queryKeyNormWeightLayers) {
|
|
749
|
-
if (!Number.isInteger(layerIdx) || layerIdx < 0) {
|
|
750
|
-
errors.push('attention.queryKeyNormWeightLayers must contain only non-negative integer layer indices');
|
|
751
|
-
break;
|
|
752
|
-
}
|
|
753
|
-
}
|
|
754
|
-
}
|
|
755
|
-
}
|
|
756
|
-
if (inf.attention.valueNorm == null) {
|
|
757
|
-
errors.push('attention.valueNorm is required');
|
|
758
|
-
}
|
|
759
|
-
if (inf.attention.attentionBias == null) {
|
|
760
|
-
errors.push('attention.attentionBias is required');
|
|
761
|
-
}
|
|
762
|
-
if (inf.attention.causal == null) {
|
|
763
|
-
errors.push('attention.causal is required');
|
|
764
|
-
}
|
|
765
|
-
// Attention fields - nullable required (undefined = missing, null = disabled)
|
|
766
|
-
if (inf.attention.slidingWindow === undefined) {
|
|
767
|
-
errors.push('attention.slidingWindow must be explicitly set (null for no sliding window, or number)');
|
|
768
|
-
}
|
|
769
|
-
if (inf.attention.attnLogitSoftcapping === undefined) {
|
|
770
|
-
errors.push('attention.attnLogitSoftcapping must be explicitly set (null for no softcapping, or number)');
|
|
771
|
-
}
|
|
772
|
-
|
|
773
|
-
// Normalization fields
|
|
774
|
-
if (
|
|
775
|
-
inf.normalization.type !== undefined
|
|
776
|
-
&& !['rmsnorm', 'layernorm'].includes(inf.normalization.type)
|
|
777
|
-
) {
|
|
778
|
-
errors.push('normalization.type must be rmsnorm or layernorm');
|
|
779
|
-
}
|
|
780
|
-
if (inf.normalization.rmsNormWeightOffset == null) {
|
|
781
|
-
errors.push('normalization.rmsNormWeightOffset is required');
|
|
782
|
-
}
|
|
783
|
-
if (inf.normalization.rmsNormEps == null) {
|
|
784
|
-
errors.push('normalization.rmsNormEps is required');
|
|
785
|
-
}
|
|
786
|
-
if (inf.normalization.postAttentionNorm == null) {
|
|
787
|
-
errors.push('normalization.postAttentionNorm is required');
|
|
788
|
-
}
|
|
789
|
-
if (inf.normalization.preFeedforwardNorm == null) {
|
|
790
|
-
errors.push('normalization.preFeedforwardNorm is required');
|
|
791
|
-
}
|
|
792
|
-
if (inf.normalization.postFeedforwardNorm == null) {
|
|
793
|
-
errors.push('normalization.postFeedforwardNorm is required');
|
|
794
|
-
}
|
|
795
|
-
if (
|
|
796
|
-
inf.normalization.finalNormBiasTensor !== undefined
|
|
797
|
-
&&
|
|
798
|
-
inf.normalization.finalNormBiasTensor !== null
|
|
799
|
-
&& (
|
|
800
|
-
typeof inf.normalization.finalNormBiasTensor !== 'string'
|
|
801
|
-
|| inf.normalization.finalNormBiasTensor.trim() === ''
|
|
802
|
-
)
|
|
803
|
-
) {
|
|
804
|
-
errors.push('normalization.finalNormBiasTensor must be null or a non-empty string');
|
|
805
|
-
}
|
|
806
|
-
if (
|
|
807
|
-
(inf.normalization.type ?? 'rmsnorm') === 'rmsnorm'
|
|
808
|
-
&& (inf.normalization.finalNormBiasTensor ?? null) !== null
|
|
809
|
-
) {
|
|
810
|
-
errors.push('normalization.finalNormBiasTensor requires normalization.type="layernorm"');
|
|
811
|
-
}
|
|
812
|
-
|
|
813
|
-
// FFN fields
|
|
814
|
-
if (inf.ffn.activation == null) {
|
|
815
|
-
errors.push('ffn.activation is required');
|
|
816
|
-
}
|
|
817
|
-
if (inf.ffn.gatedActivation == null) {
|
|
818
|
-
errors.push('ffn.gatedActivation is required');
|
|
819
|
-
}
|
|
820
|
-
if (inf.ffn.branchMode == null) {
|
|
821
|
-
errors.push('ffn.branchMode is required');
|
|
822
|
-
} else {
|
|
823
|
-
const normalizedBranchMode = typeof inf.ffn.branchMode === 'string'
|
|
824
|
-
? inf.ffn.branchMode.trim().toLowerCase()
|
|
825
|
-
: '';
|
|
826
|
-
const supportedBranchModes = new Set(['auto', 'dense', 'moe', 'dense_plus_moe']);
|
|
827
|
-
if (!supportedBranchModes.has(normalizedBranchMode)) {
|
|
828
|
-
errors.push('ffn.branchMode must be one of: auto, dense, moe, dense_plus_moe');
|
|
829
|
-
}
|
|
830
|
-
}
|
|
831
|
-
if (inf.ffn.useDoubleWideMlp == null) {
|
|
832
|
-
errors.push('ffn.useDoubleWideMlp is required');
|
|
833
|
-
}
|
|
834
|
-
if (inf.ffn.swigluLimit === undefined) {
|
|
835
|
-
errors.push('ffn.swigluLimit must be explicitly set (null for no limit, or number)');
|
|
836
|
-
} else {
|
|
837
|
-
const limit = inf.ffn.swigluLimit;
|
|
838
|
-
if (limit !== null && (typeof limit !== 'number' || Number.isNaN(limit) || limit <= 0)) {
|
|
839
|
-
errors.push('ffn.swigluLimit must be a positive number or null');
|
|
840
|
-
}
|
|
841
|
-
}
|
|
842
|
-
|
|
843
|
-
// RoPE fields - non-nullable required
|
|
844
|
-
if (inf.rope.ropeTheta == null) {
|
|
845
|
-
errors.push('rope.ropeTheta is required');
|
|
846
|
-
}
|
|
847
|
-
if (inf.rope.ropeScalingFactor == null) {
|
|
848
|
-
errors.push('rope.ropeScalingFactor is required (use 1.0 for no scaling)');
|
|
849
|
-
}
|
|
850
|
-
// RoPE fields - nullable required (undefined = missing, null = disabled)
|
|
851
|
-
if (inf.rope.ropeScalingType === undefined) {
|
|
852
|
-
errors.push('rope.ropeScalingType must be explicitly set (null for no scaling, or scaling type string)');
|
|
853
|
-
}
|
|
854
|
-
if (inf.rope.ropeLocalTheta === undefined) {
|
|
855
|
-
errors.push('rope.ropeLocalTheta must be explicitly set (null for no local theta, or number)');
|
|
856
|
-
}
|
|
857
|
-
if (
|
|
858
|
-
inf.rope.ropeInterleaved !== undefined
|
|
859
|
-
&& inf.rope.ropeInterleaved != null
|
|
860
|
-
&& typeof inf.rope.ropeInterleaved !== 'boolean'
|
|
861
|
-
) {
|
|
862
|
-
errors.push('rope.ropeInterleaved must be boolean when provided');
|
|
863
|
-
}
|
|
864
|
-
if (inf.rope.mropeInterleaved == null) {
|
|
865
|
-
errors.push('rope.mropeInterleaved is required');
|
|
866
|
-
}
|
|
867
|
-
if (inf.rope.mropeSection === undefined) {
|
|
868
|
-
errors.push('rope.mropeSection must be explicitly set (null when unused, or an array of positive integers)');
|
|
869
|
-
}
|
|
870
|
-
if (inf.rope.partialRotaryFactor === undefined) {
|
|
871
|
-
errors.push('rope.partialRotaryFactor must be explicitly set (null when unused, or a number in (0, 1])');
|
|
872
|
-
} else {
|
|
873
|
-
const factor = inf.rope.partialRotaryFactor;
|
|
874
|
-
if (factor !== null && (typeof factor !== 'number' || Number.isNaN(factor) || factor <= 0 || factor > 1)) {
|
|
875
|
-
errors.push('rope.partialRotaryFactor must be a number in (0, 1] or null');
|
|
876
|
-
}
|
|
877
|
-
}
|
|
878
|
-
if (inf.rope.ropeLocalPartialRotaryFactor === undefined) {
|
|
879
|
-
errors.push('rope.ropeLocalPartialRotaryFactor must be explicitly set (null when unused, or a number in (0, 1])');
|
|
880
|
-
} else {
|
|
881
|
-
const factor = inf.rope.ropeLocalPartialRotaryFactor;
|
|
882
|
-
if (factor !== null && (typeof factor !== 'number' || Number.isNaN(factor) || factor <= 0 || factor > 1)) {
|
|
883
|
-
errors.push('rope.ropeLocalPartialRotaryFactor must be a number in (0, 1] or null');
|
|
884
|
-
}
|
|
885
|
-
}
|
|
886
|
-
if (inf.rope.ropeFrequencyBaseDim === undefined) {
|
|
887
|
-
errors.push('rope.ropeFrequencyBaseDim must be explicitly set (null when using rotary dim, or a positive even integer)');
|
|
888
|
-
} else {
|
|
889
|
-
const dim = inf.rope.ropeFrequencyBaseDim;
|
|
890
|
-
if (dim !== null && (typeof dim !== 'number' || Number.isNaN(dim) || dim <= 0 || (Math.trunc(dim) % 2) !== 0)) {
|
|
891
|
-
errors.push('rope.ropeFrequencyBaseDim must be a positive even integer or null');
|
|
892
|
-
}
|
|
893
|
-
}
|
|
894
|
-
if (inf.rope.ropeLocalFrequencyBaseDim === undefined) {
|
|
895
|
-
errors.push('rope.ropeLocalFrequencyBaseDim must be explicitly set (null when using local rotary dim, or a positive even integer)');
|
|
896
|
-
} else {
|
|
897
|
-
const dim = inf.rope.ropeLocalFrequencyBaseDim;
|
|
898
|
-
if (dim !== null && (typeof dim !== 'number' || Number.isNaN(dim) || dim <= 0 || (Math.trunc(dim) % 2) !== 0)) {
|
|
899
|
-
errors.push('rope.ropeLocalFrequencyBaseDim must be a positive even integer or null');
|
|
900
|
-
}
|
|
901
|
-
}
|
|
902
|
-
|
|
903
|
-
// Output fields - non-nullable required
|
|
904
|
-
if (inf.output.tieWordEmbeddings == null) {
|
|
905
|
-
errors.push('output.tieWordEmbeddings is required');
|
|
906
|
-
}
|
|
907
|
-
if (inf.output.scaleEmbeddings == null) {
|
|
908
|
-
errors.push('output.scaleEmbeddings is required');
|
|
909
|
-
}
|
|
910
|
-
if (inf.output.embeddingScale === undefined) {
|
|
911
|
-
errors.push('output.embeddingScale must be explicitly set (null for scaleEmbeddings semantics, or number)');
|
|
912
|
-
} else {
|
|
913
|
-
const embeddingScale = inf.output.embeddingScale;
|
|
914
|
-
if (embeddingScale !== null && (typeof embeddingScale !== 'number' || !Number.isFinite(embeddingScale) || embeddingScale <= 0)) {
|
|
915
|
-
errors.push('output.embeddingScale must be a positive finite number or null');
|
|
916
|
-
}
|
|
917
|
-
if (embeddingScale !== null && inf.output.scaleEmbeddings === true) {
|
|
918
|
-
errors.push('output.embeddingScale cannot be set when output.scaleEmbeddings is true');
|
|
919
|
-
}
|
|
920
|
-
}
|
|
921
|
-
if (inf.output.logitInputScale == null) {
|
|
922
|
-
errors.push('output.logitInputScale is required');
|
|
923
|
-
} else if (
|
|
924
|
-
typeof inf.output.logitInputScale !== 'number'
|
|
925
|
-
|| !Number.isFinite(inf.output.logitInputScale)
|
|
926
|
-
|| inf.output.logitInputScale <= 0
|
|
927
|
-
) {
|
|
928
|
-
errors.push('output.logitInputScale must be a positive finite number');
|
|
929
|
-
}
|
|
930
|
-
if (inf.output.embeddingTranspose == null) {
|
|
931
|
-
errors.push('output.embeddingTranspose is required');
|
|
932
|
-
}
|
|
933
|
-
// Output fields - nullable required (undefined = missing, null = disabled)
|
|
934
|
-
if (inf.output.finalLogitSoftcapping === undefined) {
|
|
935
|
-
errors.push('output.finalLogitSoftcapping must be explicitly set (null for no softcapping, or number)');
|
|
936
|
-
}
|
|
937
|
-
if (inf.output.embeddingVocabSize === undefined) {
|
|
938
|
-
errors.push('output.embeddingVocabSize must be explicitly set (null to use architecture.vocabSize, or number)');
|
|
939
|
-
}
|
|
940
|
-
if (inf.output.embeddingPostprocessor === undefined) {
|
|
941
|
-
errors.push('output.embeddingPostprocessor must be explicitly set (null when unused, or an object)');
|
|
942
|
-
} else if (inf.output.embeddingPostprocessor !== null) {
|
|
943
|
-
const postprocessor = inf.output.embeddingPostprocessor;
|
|
944
|
-
if (!postprocessor || typeof postprocessor !== 'object' || Array.isArray(postprocessor)) {
|
|
945
|
-
errors.push('output.embeddingPostprocessor must be null or an object');
|
|
946
|
-
} else {
|
|
947
|
-
if (postprocessor.poolingMode !== 'mean' && postprocessor.poolingMode !== 'last') {
|
|
948
|
-
errors.push('output.embeddingPostprocessor.poolingMode must be "mean" or "last"');
|
|
949
|
-
}
|
|
950
|
-
if (typeof postprocessor.includePrompt !== 'boolean') {
|
|
951
|
-
errors.push('output.embeddingPostprocessor.includePrompt is required');
|
|
952
|
-
}
|
|
953
|
-
if (!Array.isArray(postprocessor.projections)) {
|
|
954
|
-
errors.push('output.embeddingPostprocessor.projections must be an array');
|
|
955
|
-
} else {
|
|
956
|
-
for (let i = 0; i < postprocessor.projections.length; i++) {
|
|
957
|
-
const projection = postprocessor.projections[i];
|
|
958
|
-
const prefix = `output.embeddingPostprocessor.projections[${i}]`;
|
|
959
|
-
if (typeof projection?.weightTensor !== 'string' || projection.weightTensor.trim() === '') {
|
|
960
|
-
errors.push(`${prefix}.weightTensor is required`);
|
|
961
|
-
}
|
|
962
|
-
if (projection?.biasTensor === undefined) {
|
|
963
|
-
errors.push(`${prefix}.biasTensor must be explicitly set (null when unused, or tensor name)`);
|
|
964
|
-
} else if (projection.biasTensor !== null && (typeof projection.biasTensor !== 'string' || projection.biasTensor.trim() === '')) {
|
|
965
|
-
errors.push(`${prefix}.biasTensor must be null or a non-empty string`);
|
|
966
|
-
}
|
|
967
|
-
if (!Number.isFinite(projection?.inputSize) || projection.inputSize <= 0) {
|
|
968
|
-
errors.push(`${prefix}.inputSize must be a positive number`);
|
|
969
|
-
}
|
|
970
|
-
if (!Number.isFinite(projection?.outputSize) || projection.outputSize <= 0) {
|
|
971
|
-
errors.push(`${prefix}.outputSize must be a positive number`);
|
|
972
|
-
}
|
|
973
|
-
if (projection?.activation !== 'identity') {
|
|
974
|
-
errors.push(`${prefix}.activation must be "identity"`);
|
|
975
|
-
}
|
|
976
|
-
}
|
|
977
|
-
}
|
|
978
|
-
if (postprocessor.normalize === undefined) {
|
|
979
|
-
errors.push('output.embeddingPostprocessor.normalize must be explicitly set (null when unused, or "l2")');
|
|
980
|
-
} else if (postprocessor.normalize !== null && postprocessor.normalize !== 'l2') {
|
|
981
|
-
errors.push('output.embeddingPostprocessor.normalize must be null or "l2"');
|
|
982
|
-
}
|
|
983
|
-
}
|
|
984
|
-
}
|
|
985
|
-
if (
|
|
986
|
-
inf.output.lmHeadBiasTensor !== undefined
|
|
987
|
-
&& inf.output.lmHeadBiasTensor !== null
|
|
988
|
-
&& (typeof inf.output.lmHeadBiasTensor !== 'string' || inf.output.lmHeadBiasTensor.trim() === '')
|
|
989
|
-
) {
|
|
990
|
-
errors.push('output.lmHeadBiasTensor must be null or a non-empty tensor name');
|
|
991
|
-
}
|
|
992
|
-
|
|
993
|
-
if (inf.supportsSequence !== undefined && typeof inf.supportsSequence !== 'boolean') {
|
|
994
|
-
errors.push('supportsSequence must be boolean when provided');
|
|
995
|
-
}
|
|
996
|
-
if (inf.sequence !== undefined && inf.sequence !== null) {
|
|
997
|
-
const sequence = inf.sequence;
|
|
998
|
-
if (!sequence || typeof sequence !== 'object' || Array.isArray(sequence)) {
|
|
999
|
-
errors.push('sequence must be null or an object');
|
|
1000
|
-
} else {
|
|
1001
|
-
if (sequence.alphabet !== 'amino_acid' && sequence.alphabet !== 'nucleotide') {
|
|
1002
|
-
errors.push('sequence.alphabet must be "amino_acid" or "nucleotide"');
|
|
1003
|
-
}
|
|
1004
|
-
if (typeof sequence.tokenEmbeddings !== 'boolean') {
|
|
1005
|
-
errors.push('sequence.tokenEmbeddings must be boolean');
|
|
1006
|
-
}
|
|
1007
|
-
if (typeof sequence.logits !== 'boolean') {
|
|
1008
|
-
errors.push('sequence.logits must be boolean');
|
|
1009
|
-
}
|
|
1010
|
-
if (sequence.pooledEmbedding !== null) {
|
|
1011
|
-
const pooling = sequence.pooledEmbedding;
|
|
1012
|
-
if (!pooling || typeof pooling !== 'object' || Array.isArray(pooling)) {
|
|
1013
|
-
errors.push('sequence.pooledEmbedding must be null or an object');
|
|
1014
|
-
} else {
|
|
1015
|
-
if (pooling.mode !== 'mean' && pooling.mode !== 'last') {
|
|
1016
|
-
errors.push('sequence.pooledEmbedding.mode must be "mean" or "last"');
|
|
1017
|
-
}
|
|
1018
|
-
if (!Array.isArray(pooling.excludeTokenIds) || pooling.excludeTokenIds.some(
|
|
1019
|
-
(tokenId) => !Number.isInteger(tokenId) || tokenId < 0
|
|
1020
|
-
)) {
|
|
1021
|
-
errors.push('sequence.pooledEmbedding.excludeTokenIds must be an array of non-negative integers');
|
|
1022
|
-
}
|
|
1023
|
-
}
|
|
1024
|
-
}
|
|
1025
|
-
}
|
|
1026
|
-
}
|
|
1027
|
-
if (inf.supportsSequence === true && (inf.sequence == null || typeof inf.sequence !== 'object')) {
|
|
1028
|
-
errors.push('sequence is required when supportsSequence=true');
|
|
1029
|
-
}
|
|
1030
|
-
if (inf.supportsSequence !== true && inf.sequence != null) {
|
|
1031
|
-
errors.push('supportsSequence must be true when sequence is configured');
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
// Layer pattern fields
|
|
1035
|
-
if (inf.layerPattern?.type == null) {
|
|
1036
|
-
errors.push('layerPattern.type is required');
|
|
1037
|
-
}
|
|
1038
|
-
if (inf.layerPattern?.globalPattern === undefined) {
|
|
1039
|
-
errors.push('layerPattern.globalPattern must be explicitly set (null if not applicable)');
|
|
1040
|
-
}
|
|
1041
|
-
if (inf.layerPattern?.period === undefined) {
|
|
1042
|
-
errors.push('layerPattern.period must be explicitly set (null if not applicable)');
|
|
1043
|
-
}
|
|
1044
|
-
if (inf.layerPattern?.offset === undefined) {
|
|
1045
|
-
errors.push('layerPattern.offset must be explicitly set (null if not applicable)');
|
|
1046
|
-
}
|
|
1047
|
-
if (inf.layerPattern?.residualBranchScale == null) {
|
|
1048
|
-
errors.push('layerPattern.residualBranchScale is required');
|
|
1049
|
-
} else if (
|
|
1050
|
-
typeof inf.layerPattern.residualBranchScale !== 'number'
|
|
1051
|
-
|| !Number.isFinite(inf.layerPattern.residualBranchScale)
|
|
1052
|
-
|| inf.layerPattern.residualBranchScale <= 0
|
|
1053
|
-
) {
|
|
1054
|
-
errors.push('layerPattern.residualBranchScale must be a positive finite number');
|
|
1055
|
-
}
|
|
1056
|
-
if (inf.layerPattern?.type === 'custom' && inf.layerPattern?.layerTypes === undefined) {
|
|
1057
|
-
errors.push('layerPattern.layerTypes must be explicitly set for custom patterns');
|
|
1058
|
-
}
|
|
1059
|
-
|
|
1060
|
-
// Chat template fields
|
|
1061
|
-
if (inf.chatTemplate?.type === undefined) {
|
|
1062
|
-
errors.push('chatTemplate.type must be explicitly set (null for no template)');
|
|
1063
|
-
}
|
|
1064
|
-
if (inf.chatTemplate?.enabled == null) {
|
|
1065
|
-
errors.push('chatTemplate.enabled is required');
|
|
1066
|
-
}
|
|
1067
|
-
|
|
1068
|
-
// RoPE YARN fields
|
|
1069
|
-
if (inf.rope.yarnBetaFast === undefined) {
|
|
1070
|
-
errors.push('rope.yarnBetaFast must be explicitly set (null if not YARN)');
|
|
1071
|
-
}
|
|
1072
|
-
if (inf.rope.yarnBetaSlow === undefined) {
|
|
1073
|
-
errors.push('rope.yarnBetaSlow must be explicitly set (null if not YARN)');
|
|
1074
|
-
}
|
|
1075
|
-
if (inf.rope.yarnOriginalMaxPos === undefined) {
|
|
1076
|
-
errors.push('rope.yarnOriginalMaxPos must be explicitly set (null if not YARN)');
|
|
1077
|
-
}
|
|
1078
|
-
if (inf.rope.longropeShortFactor === undefined) {
|
|
1079
|
-
errors.push('rope.longropeShortFactor must be explicitly set (null if not LongRoPE)');
|
|
1080
|
-
}
|
|
1081
|
-
if (inf.rope.longropeLongFactor === undefined) {
|
|
1082
|
-
errors.push('rope.longropeLongFactor must be explicitly set (null if not LongRoPE)');
|
|
1083
|
-
}
|
|
1084
|
-
if (inf.rope.longropeOriginalMaxPos === undefined) {
|
|
1085
|
-
errors.push('rope.longropeOriginalMaxPos must be explicitly set (null if not LongRoPE)');
|
|
1086
|
-
}
|
|
1087
|
-
if (inf.rope.ropeScalingType === 'longrope') {
|
|
1088
|
-
if (!Array.isArray(inf.rope.longropeShortFactor) || inf.rope.longropeShortFactor.length === 0) {
|
|
1089
|
-
errors.push('rope.longropeShortFactor must be a non-empty number array for LongRoPE');
|
|
1090
|
-
}
|
|
1091
|
-
if (!Array.isArray(inf.rope.longropeLongFactor) || inf.rope.longropeLongFactor.length === 0) {
|
|
1092
|
-
errors.push('rope.longropeLongFactor must be a non-empty number array for LongRoPE');
|
|
1093
|
-
}
|
|
1094
|
-
if (
|
|
1095
|
-
typeof inf.rope.longropeOriginalMaxPos !== 'number'
|
|
1096
|
-
|| !Number.isFinite(inf.rope.longropeOriginalMaxPos)
|
|
1097
|
-
|| inf.rope.longropeOriginalMaxPos <= 0
|
|
1098
|
-
) {
|
|
1099
|
-
errors.push('rope.longropeOriginalMaxPos must be a positive finite number for LongRoPE');
|
|
1100
|
-
}
|
|
1101
|
-
} else if (
|
|
1102
|
-
inf.rope.longropeShortFactor !== null
|
|
1103
|
-
|| inf.rope.longropeLongFactor !== null
|
|
1104
|
-
|| inf.rope.longropeOriginalMaxPos !== null
|
|
1105
|
-
) {
|
|
1106
|
-
errors.push('rope LongRoPE fields must be null unless rope.ropeScalingType is "longrope"');
|
|
1107
|
-
}
|
|
1108
|
-
if (inf.rope.ropeLocalYarnBetaFast === undefined) {
|
|
1109
|
-
errors.push('rope.ropeLocalYarnBetaFast must be explicitly set (null if not local YARN)');
|
|
1110
|
-
}
|
|
1111
|
-
if (inf.rope.ropeLocalYarnBetaSlow === undefined) {
|
|
1112
|
-
errors.push('rope.ropeLocalYarnBetaSlow must be explicitly set (null if not local YARN)');
|
|
1113
|
-
}
|
|
1114
|
-
if (inf.rope.ropeLocalYarnOriginalMaxPos === undefined) {
|
|
1115
|
-
errors.push('rope.ropeLocalYarnOriginalMaxPos must be explicitly set (null if not local YARN)');
|
|
1116
|
-
}
|
|
1117
|
-
|
|
1118
|
-
if (errors.length > 0) {
|
|
1119
|
-
throw new Error(
|
|
1120
|
-
`Manifest "${modelId}" has incomplete inference config. ` +
|
|
1121
|
-
`Missing required fields:\n - ${errors.join('\n - ')}\n` +
|
|
1122
|
-
`Re-convert the model using the latest converter.`
|
|
1123
|
-
);
|
|
1124
|
-
}
|
|
1125
|
-
}
|
|
1126
|
-
|
|
1127
213
|
function normalizeLayerTypeTag(value) {
|
|
1128
214
|
const normalized = typeof value === 'string' ? value.trim().toLowerCase() : '';
|
|
1129
215
|
if (!normalized) return null;
|
|
@@ -1166,306 +252,6 @@ function normalizeLayerTypeTag(value) {
|
|
|
1166
252
|
return null;
|
|
1167
253
|
}
|
|
1168
254
|
|
|
1169
|
-
function resolveVisionConfig(rawConfig, manifest) {
|
|
1170
|
-
const vc = rawConfig?.vision_config ?? manifest?.config?.vision_config;
|
|
1171
|
-
if (!vc || typeof vc !== 'object') {
|
|
1172
|
-
log.debug(
|
|
1173
|
-
'Config',
|
|
1174
|
-
`Vision config not present for model "${manifest?.modelId ?? 'unknown'}"; vision pipeline disabled.`
|
|
1175
|
-
);
|
|
1176
|
-
return null;
|
|
1177
|
-
}
|
|
1178
|
-
const modelId = manifest?.modelId ?? 'unknown';
|
|
1179
|
-
const resolveRequiredVisionField = (keys, label) => {
|
|
1180
|
-
for (const key of keys) {
|
|
1181
|
-
if (vc[key] !== undefined) {
|
|
1182
|
-
return vc[key];
|
|
1183
|
-
}
|
|
1184
|
-
}
|
|
1185
|
-
throw new Error(
|
|
1186
|
-
`Manifest "${modelId}" is missing vision_config.${label}. ` +
|
|
1187
|
-
'Re-convert the model with explicit vision config metadata.'
|
|
1188
|
-
);
|
|
1189
|
-
};
|
|
1190
|
-
const resolveRequiredPositiveInteger = (keys, label) => {
|
|
1191
|
-
const value = resolveRequiredVisionField(keys, label);
|
|
1192
|
-
const number = Number(value);
|
|
1193
|
-
if (!Number.isFinite(number) || number <= 0 || Math.floor(number) !== number) {
|
|
1194
|
-
throw new Error(
|
|
1195
|
-
`Manifest "${modelId}" has invalid vision_config.${label}=${JSON.stringify(value)}. ` +
|
|
1196
|
-
'Expected a positive integer.'
|
|
1197
|
-
);
|
|
1198
|
-
}
|
|
1199
|
-
return Math.trunc(number);
|
|
1200
|
-
};
|
|
1201
|
-
const resolveRequiredNonNegativeInteger = (keys, label) => {
|
|
1202
|
-
const value = resolveRequiredVisionField(keys, label);
|
|
1203
|
-
const number = Number(value);
|
|
1204
|
-
if (!Number.isFinite(number) || number < 0 || Math.floor(number) !== number) {
|
|
1205
|
-
throw new Error(
|
|
1206
|
-
`Manifest "${modelId}" has invalid vision_config.${label}=${JSON.stringify(value)}. ` +
|
|
1207
|
-
'Expected a non-negative integer.'
|
|
1208
|
-
);
|
|
1209
|
-
}
|
|
1210
|
-
return Math.trunc(number);
|
|
1211
|
-
};
|
|
1212
|
-
const resolveRequiredPositiveNumber = (value, label) => {
|
|
1213
|
-
const number = Number(value);
|
|
1214
|
-
if (!Number.isFinite(number) || number <= 0) {
|
|
1215
|
-
throw new Error(
|
|
1216
|
-
`Manifest "${modelId}" has invalid ${label}=${JSON.stringify(value)}. ` +
|
|
1217
|
-
'Expected a positive number.'
|
|
1218
|
-
);
|
|
1219
|
-
}
|
|
1220
|
-
return number;
|
|
1221
|
-
};
|
|
1222
|
-
const visionArchitecture = String(resolveRequiredVisionField(['vision_architecture'], 'vision_architecture')).trim();
|
|
1223
|
-
if (visionArchitecture !== 'gemma4' && visionArchitecture !== 'qwen3vl') {
|
|
1224
|
-
throw new Error(
|
|
1225
|
-
`Manifest "${modelId}" has unsupported vision_config.vision_architecture="${visionArchitecture}". ` +
|
|
1226
|
-
'Supported: "gemma4", "qwen3vl".'
|
|
1227
|
-
);
|
|
1228
|
-
}
|
|
1229
|
-
|
|
1230
|
-
if (visionArchitecture === 'gemma4') {
|
|
1231
|
-
const depth = resolveRequiredNonNegativeInteger(['depth', 'num_hidden_layers'], 'num_hidden_layers');
|
|
1232
|
-
const isEncoderFree = (depth === 0);
|
|
1233
|
-
|
|
1234
|
-
const hiddenSize = resolveRequiredPositiveInteger(['hidden_size'], 'hidden_size');
|
|
1235
|
-
const numHeads = isEncoderFree
|
|
1236
|
-
? (vc.num_heads !== undefined || vc.num_attention_heads !== undefined ? resolveRequiredPositiveInteger(['num_heads', 'num_attention_heads'], 'num_attention_heads') : 1)
|
|
1237
|
-
: resolveRequiredPositiveInteger(['num_heads', 'num_attention_heads'], 'num_attention_heads');
|
|
1238
|
-
const ropeParameters = vc.rope_parameters;
|
|
1239
|
-
if (!ropeParameters || typeof ropeParameters !== 'object') {
|
|
1240
|
-
if (!isEncoderFree) {
|
|
1241
|
-
throw new Error(
|
|
1242
|
-
`Manifest "${modelId}" is missing vision_config.rope_parameters. ` +
|
|
1243
|
-
'Re-convert the model with explicit Gemma 4 vision RoPE metadata.'
|
|
1244
|
-
);
|
|
1245
|
-
}
|
|
1246
|
-
}
|
|
1247
|
-
const hiddenActivation = String(resolveRequiredVisionField(['hidden_activation'], 'hidden_activation')).trim();
|
|
1248
|
-
if (hiddenActivation !== 'gelu' && hiddenActivation !== 'gelu_pytorch_tanh') {
|
|
1249
|
-
throw new Error(
|
|
1250
|
-
`Manifest "${modelId}" has unsupported Gemma 4 vision hidden_activation="${hiddenActivation}". ` +
|
|
1251
|
-
'Supported values: "gelu", "gelu_pytorch_tanh".'
|
|
1252
|
-
);
|
|
1253
|
-
}
|
|
1254
|
-
if (vc.standardize === true) {
|
|
1255
|
-
throw new Error(
|
|
1256
|
-
`Manifest "${modelId}" enables vision_config.standardize, but Gemma 4 runtime preprocessing does not support it yet.`
|
|
1257
|
-
);
|
|
1258
|
-
}
|
|
1259
|
-
if (vc.use_clipped_linears !== true) {
|
|
1260
|
-
throw new Error(
|
|
1261
|
-
`Manifest "${modelId}" requires vision_config.use_clipped_linears=true for Gemma 4 vision weights.`
|
|
1262
|
-
);
|
|
1263
|
-
}
|
|
1264
|
-
|
|
1265
|
-
return {
|
|
1266
|
-
depth,
|
|
1267
|
-
hiddenSize,
|
|
1268
|
-
intermediateSize: isEncoderFree
|
|
1269
|
-
? (vc.intermediate_size !== undefined ? resolveRequiredPositiveInteger(['intermediate_size'], 'intermediate_size') : 1)
|
|
1270
|
-
: resolveRequiredPositiveInteger(['intermediate_size'], 'intermediate_size'),
|
|
1271
|
-
numHeads,
|
|
1272
|
-
numKeyValueHeads: isEncoderFree
|
|
1273
|
-
? (vc.num_key_value_heads !== undefined ? resolveRequiredPositiveInteger(['num_key_value_heads'], 'num_key_value_heads') : 1)
|
|
1274
|
-
: resolveRequiredPositiveInteger(['num_key_value_heads'], 'num_key_value_heads'),
|
|
1275
|
-
headDim: isEncoderFree
|
|
1276
|
-
? (vc.head_dim !== undefined || vc.global_head_dim !== undefined ? resolveRequiredPositiveInteger(['head_dim', 'global_head_dim'], 'head_dim') : 1)
|
|
1277
|
-
: resolveRequiredPositiveInteger(['head_dim', 'global_head_dim'], 'head_dim'),
|
|
1278
|
-
outHiddenSize: vc.out_hidden_size ?? vc.output_proj_dims ?? null,
|
|
1279
|
-
patchSize: resolveRequiredPositiveInteger(['patch_size'], 'patch_size'),
|
|
1280
|
-
poolingKernelSize: resolveRequiredPositiveInteger(['pooling_kernel_size'], 'pooling_kernel_size'),
|
|
1281
|
-
spatialMergeSize: vc.spatial_merge_size ?? null,
|
|
1282
|
-
temporalPatchSize: vc.temporal_patch_size ?? null,
|
|
1283
|
-
positionEmbeddingSize: resolveRequiredPositiveInteger(['position_embedding_size'], 'position_embedding_size'),
|
|
1284
|
-
defaultOutputLength: resolveRequiredPositiveInteger(['default_output_length'], 'default_output_length'),
|
|
1285
|
-
ropeTheta: isEncoderFree && (!ropeParameters || ropeParameters.rope_theta === undefined)
|
|
1286
|
-
? 10000
|
|
1287
|
-
: resolveRequiredPositiveNumber(ropeParameters.rope_theta, 'vision_config.rope_parameters.rope_theta'),
|
|
1288
|
-
eps: resolveRequiredPositiveNumber(
|
|
1289
|
-
resolveRequiredVisionField(['eps', 'rms_norm_eps'], 'rms_norm_eps'),
|
|
1290
|
-
'vision_config.rms_norm_eps'
|
|
1291
|
-
),
|
|
1292
|
-
hiddenActivation,
|
|
1293
|
-
standardize: false,
|
|
1294
|
-
useClippedLinears: true,
|
|
1295
|
-
deepstackVisualIndexes: [],
|
|
1296
|
-
imageTokenId: rawConfig?.image_token_id ?? manifest?.image_token_id ?? null,
|
|
1297
|
-
visionArchitecture,
|
|
1298
|
-
softTokenBudgetTiers: Array.isArray(vc.soft_token_budget_tiers)
|
|
1299
|
-
? vc.soft_token_budget_tiers.map(Number).filter((n) => Number.isFinite(n) && n > 0)
|
|
1300
|
-
: [70, 140, 280, 560, 1120],
|
|
1301
|
-
};
|
|
1302
|
-
}
|
|
1303
|
-
|
|
1304
|
-
const hiddenSize = resolveRequiredPositiveInteger(['hidden_size'], 'hidden_size');
|
|
1305
|
-
const intermediateSize = resolveRequiredPositiveInteger(['intermediate_size'], 'intermediate_size');
|
|
1306
|
-
const numHeads = resolveRequiredPositiveInteger(['num_heads', 'num_attention_heads'], 'num_heads');
|
|
1307
|
-
const outHiddenSize = resolveRequiredPositiveInteger(['out_hidden_size', 'output_proj_dims'], 'out_hidden_size');
|
|
1308
|
-
const patchSize = resolveRequiredPositiveInteger(['patch_size'], 'patch_size');
|
|
1309
|
-
const spatialMergeSize = resolveRequiredPositiveInteger(['spatial_merge_size', 'merge_size'], 'spatial_merge_size');
|
|
1310
|
-
const temporalPatchSize = resolveRequiredPositiveInteger(['temporal_patch_size'], 'temporal_patch_size');
|
|
1311
|
-
const eps = resolveRequiredPositiveNumber(
|
|
1312
|
-
resolveRequiredVisionField(['eps', 'rms_norm_eps'], 'eps'),
|
|
1313
|
-
'vision_config.eps'
|
|
1314
|
-
);
|
|
1315
|
-
const hiddenActivation = String(
|
|
1316
|
-
resolveRequiredVisionField(['hidden_activation', 'hidden_act'], 'hidden_activation')
|
|
1317
|
-
).trim();
|
|
1318
|
-
const minPixels = resolveRequiredPositiveInteger(['min_pixels'], 'min_pixels');
|
|
1319
|
-
const maxPixels = resolveRequiredPositiveInteger(['max_pixels'], 'max_pixels');
|
|
1320
|
-
const normalization = vc.normalization;
|
|
1321
|
-
if (!normalization || typeof normalization !== 'object') {
|
|
1322
|
-
throw new Error(
|
|
1323
|
-
`Manifest "${modelId}" is missing vision_config.normalization. ` +
|
|
1324
|
-
'Re-convert the model with explicit normalization metadata.'
|
|
1325
|
-
);
|
|
1326
|
-
}
|
|
1327
|
-
if (!Array.isArray(normalization.mean) || normalization.mean.length !== 3) {
|
|
1328
|
-
throw new Error(
|
|
1329
|
-
`Manifest "${modelId}" requires vision_config.normalization.mean to contain exactly 3 values.`
|
|
1330
|
-
);
|
|
1331
|
-
}
|
|
1332
|
-
if (!Array.isArray(normalization.std) || normalization.std.length !== 3) {
|
|
1333
|
-
throw new Error(
|
|
1334
|
-
`Manifest "${modelId}" requires vision_config.normalization.std to contain exactly 3 values.`
|
|
1335
|
-
);
|
|
1336
|
-
}
|
|
1337
|
-
return {
|
|
1338
|
-
depth: resolveRequiredPositiveInteger(['depth', 'num_hidden_layers'], 'depth'),
|
|
1339
|
-
hiddenSize,
|
|
1340
|
-
intermediateSize,
|
|
1341
|
-
numHeads,
|
|
1342
|
-
numKeyValueHeads: resolveRequiredPositiveInteger(['num_key_value_heads'], 'num_key_value_heads'),
|
|
1343
|
-
headDim: resolveRequiredPositiveInteger(['head_dim'], 'head_dim'),
|
|
1344
|
-
outHiddenSize,
|
|
1345
|
-
patchSize,
|
|
1346
|
-
poolingKernelSize: resolveRequiredPositiveInteger(['pooling_kernel_size'], 'pooling_kernel_size'),
|
|
1347
|
-
spatialMergeSize,
|
|
1348
|
-
temporalPatchSize,
|
|
1349
|
-
positionEmbeddingSize: vc.position_embedding_size ?? null,
|
|
1350
|
-
defaultOutputLength: vc.default_output_length ?? null,
|
|
1351
|
-
ropeTheta: vc.rope_parameters?.rope_theta ?? null,
|
|
1352
|
-
eps,
|
|
1353
|
-
hiddenActivation,
|
|
1354
|
-
standardize: vc.standardize === true,
|
|
1355
|
-
useClippedLinears: vc.use_clipped_linears === true,
|
|
1356
|
-
minPixels,
|
|
1357
|
-
maxPixels,
|
|
1358
|
-
normalization,
|
|
1359
|
-
deepstackVisualIndexes: Array.isArray(vc.deepstack_visual_indexes) ? vc.deepstack_visual_indexes : [],
|
|
1360
|
-
imageTokenId: rawConfig?.image_token_id ?? manifest?.image_token_id ?? null,
|
|
1361
|
-
visionArchitecture,
|
|
1362
|
-
};
|
|
1363
|
-
}
|
|
1364
|
-
|
|
1365
|
-
function resolveAudioConfig(rawConfig, manifest) {
|
|
1366
|
-
const ac = rawConfig?.audio_config ?? manifest?.config?.audio_config;
|
|
1367
|
-
if (!ac || typeof ac !== 'object') {
|
|
1368
|
-
log.debug(
|
|
1369
|
-
'Config',
|
|
1370
|
-
`Audio config not present for model "${manifest?.modelId ?? 'unknown'}"; audio pipeline disabled.`
|
|
1371
|
-
);
|
|
1372
|
-
return null;
|
|
1373
|
-
}
|
|
1374
|
-
const modelId = manifest?.modelId ?? 'unknown';
|
|
1375
|
-
const resolveRequiredPositiveInteger = (value, label) => {
|
|
1376
|
-
const number = Number(value);
|
|
1377
|
-
if (!Number.isFinite(number) || number <= 0 || Math.floor(number) !== number) {
|
|
1378
|
-
throw new Error(
|
|
1379
|
-
`Manifest "${modelId}" has invalid audio_config.${label}=${JSON.stringify(value)}. ` +
|
|
1380
|
-
'Expected a positive integer.'
|
|
1381
|
-
);
|
|
1382
|
-
}
|
|
1383
|
-
return Math.trunc(number);
|
|
1384
|
-
};
|
|
1385
|
-
const resolveRequiredNonNegativeInteger = (value, label) => {
|
|
1386
|
-
const number = Number(value);
|
|
1387
|
-
if (!Number.isFinite(number) || number < 0 || Math.floor(number) !== number) {
|
|
1388
|
-
throw new Error(
|
|
1389
|
-
`Manifest "${modelId}" has invalid audio_config.${label}=${JSON.stringify(value)}. ` +
|
|
1390
|
-
'Expected a non-negative integer.'
|
|
1391
|
-
);
|
|
1392
|
-
}
|
|
1393
|
-
return Math.trunc(number);
|
|
1394
|
-
};
|
|
1395
|
-
const resolveRequiredPositiveNumber = (value, label) => {
|
|
1396
|
-
const number = Number(value);
|
|
1397
|
-
if (!Number.isFinite(number) || number <= 0) {
|
|
1398
|
-
throw new Error(
|
|
1399
|
-
`Manifest "${modelId}" has invalid audio_config.${label}=${JSON.stringify(value)}. ` +
|
|
1400
|
-
'Expected a positive number.'
|
|
1401
|
-
);
|
|
1402
|
-
}
|
|
1403
|
-
return number;
|
|
1404
|
-
};
|
|
1405
|
-
const resolveRequiredFiniteNumber = (value, label) => {
|
|
1406
|
-
const number = Number(value);
|
|
1407
|
-
if (!Number.isFinite(number)) {
|
|
1408
|
-
throw new Error(
|
|
1409
|
-
`Manifest "${modelId}" has invalid audio_config.${label}=${JSON.stringify(value)}. ` +
|
|
1410
|
-
'Expected a finite number.'
|
|
1411
|
-
);
|
|
1412
|
-
}
|
|
1413
|
-
return number;
|
|
1414
|
-
};
|
|
1415
|
-
const resolveRequiredString = (value, label) => {
|
|
1416
|
-
if (typeof value !== 'string' || value.trim().length === 0) {
|
|
1417
|
-
throw new Error(
|
|
1418
|
-
`Manifest "${modelId}" has invalid audio_config.${label}=${JSON.stringify(value)}. ` +
|
|
1419
|
-
'Expected a non-empty string.'
|
|
1420
|
-
);
|
|
1421
|
-
}
|
|
1422
|
-
return value.trim();
|
|
1423
|
-
};
|
|
1424
|
-
const audioArchitecture = String(ac.audio_architecture ?? '').trim();
|
|
1425
|
-
if (audioArchitecture !== 'gemma4') {
|
|
1426
|
-
throw new Error(
|
|
1427
|
-
`Manifest "${modelId}" has unsupported audio_config.audio_architecture="${audioArchitecture}". ` +
|
|
1428
|
-
'Supported: "gemma4".'
|
|
1429
|
-
);
|
|
1430
|
-
}
|
|
1431
|
-
|
|
1432
|
-
const depth = resolveRequiredNonNegativeInteger(ac.num_hidden_layers, 'num_hidden_layers');
|
|
1433
|
-
const isEncoderFree = (depth === 0);
|
|
1434
|
-
|
|
1435
|
-
const hiddenSize = resolveRequiredPositiveInteger(ac.hidden_size, 'hidden_size');
|
|
1436
|
-
const numAttentionHeads = resolveRequiredPositiveInteger(ac.num_attention_heads, 'num_attention_heads');
|
|
1437
|
-
const headDim = Math.trunc(hiddenSize / numAttentionHeads);
|
|
1438
|
-
|
|
1439
|
-
if (!isEncoderFree) {
|
|
1440
|
-
if (!Array.isArray(ac.subsampling_conv_channels) || ac.subsampling_conv_channels.length < 1) {
|
|
1441
|
-
throw new Error(
|
|
1442
|
-
`Manifest "${modelId}" is missing audio_config.subsampling_conv_channels array.`
|
|
1443
|
-
);
|
|
1444
|
-
}
|
|
1445
|
-
}
|
|
1446
|
-
|
|
1447
|
-
return {
|
|
1448
|
-
audioArchitecture,
|
|
1449
|
-
depth,
|
|
1450
|
-
hiddenSize,
|
|
1451
|
-
numAttentionHeads,
|
|
1452
|
-
headDim,
|
|
1453
|
-
convKernelSize: resolveRequiredPositiveInteger(ac.conv_kernel_size, 'conv_kernel_size'),
|
|
1454
|
-
subsamplingConvChannels: ac.subsampling_conv_channels.map(Number),
|
|
1455
|
-
outputProjDims: resolveRequiredPositiveInteger(ac.output_proj_dims, 'output_proj_dims'),
|
|
1456
|
-
attentionContextLeft: resolveRequiredPositiveInteger(ac.attention_context_left, 'attention_context_left'),
|
|
1457
|
-
attentionContextRight: resolveRequiredNonNegativeInteger(ac.attention_context_right, 'attention_context_right'),
|
|
1458
|
-
attentionChunkSize: resolveRequiredPositiveInteger(ac.attention_chunk_size, 'attention_chunk_size'),
|
|
1459
|
-
attentionLogitCap: resolveRequiredPositiveNumber(ac.attention_logit_cap, 'attention_logit_cap'),
|
|
1460
|
-
attentionInvalidLogitsValue: resolveRequiredFiniteNumber(ac.attention_invalid_logits_value, 'attention_invalid_logits_value'),
|
|
1461
|
-
residualWeight: resolveRequiredPositiveNumber(ac.residual_weight, 'residual_weight'),
|
|
1462
|
-
rmsNormEps: resolveRequiredPositiveNumber(ac.rms_norm_eps, 'rms_norm_eps'),
|
|
1463
|
-
hiddenAct: resolveRequiredString(ac.hidden_act, 'hidden_act'),
|
|
1464
|
-
useClippedLinears: ac.use_clipped_linears === true,
|
|
1465
|
-
audioTokenId: rawConfig?.audio_token_id ?? manifest?.audio_token_id ?? null,
|
|
1466
|
-
};
|
|
1467
|
-
}
|
|
1468
|
-
|
|
1469
255
|
function parseCustomLayerTypes(layerTypes, numLayers, modelId) {
|
|
1470
256
|
if (!Array.isArray(layerTypes) || layerTypes.length === 0) {
|
|
1471
257
|
throw new Error(
|
|
@@ -1509,7 +295,6 @@ function parseLinearNormMode(value, sharedFlag = null, modelId = 'unknown') {
|
|
|
1509
295
|
return null;
|
|
1510
296
|
}
|
|
1511
297
|
|
|
1512
|
-
|
|
1513
298
|
function toParsedConfigFromMerged(merged, manifest) {
|
|
1514
299
|
const mergedConfig = merged?.config ?? manifest.config ?? {};
|
|
1515
300
|
const rawConfig = mergedConfig.text_config ?? mergedConfig;
|
|
@@ -1631,7 +416,6 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1631
416
|
}
|
|
1632
417
|
}
|
|
1633
418
|
}
|
|
1634
|
-
|
|
1635
419
|
if (!Array.isArray(layerTypes) && Array.isArray(config.layer_types) && config.layer_types.length > 0) {
|
|
1636
420
|
layerTypes = parseCustomLayerTypes(config.layer_types, arch.numLayers, merged.modelId);
|
|
1637
421
|
}
|
|
@@ -1886,7 +670,8 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1886
670
|
const embeddingScale = inf.output.embeddingScale;
|
|
1887
671
|
const logitInputScale = inf.output.logitInputScale;
|
|
1888
672
|
const residualBranchScale = inf.layerPattern.residualBranchScale;
|
|
1889
|
-
|
|
673
|
+
const heterogeneousAttention = resolveHeterogeneousAttentionContract(inf, arch.numLayers, merged.modelId);
|
|
674
|
+
const postNormContract = resolvePostNormContract(inf.normalization);
|
|
1890
675
|
return {
|
|
1891
676
|
modelType: manifest.modelType,
|
|
1892
677
|
numLayers: arch.numLayers,
|
|
@@ -1933,12 +718,15 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1933
718
|
finalNormBiasTensor: inf.normalization.finalNormBiasTensor ?? null,
|
|
1934
719
|
rmsNormEps: inf.normalization.rmsNormEps,
|
|
1935
720
|
rmsNormWeightOffset: inf.normalization.rmsNormWeightOffset,
|
|
721
|
+
...postNormContract,
|
|
1936
722
|
postAttentionNorm: inf.normalization.postAttentionNorm,
|
|
1937
723
|
preFeedforwardNorm: inf.normalization.preFeedforwardNorm,
|
|
1938
724
|
postFeedforwardNorm: inf.normalization.postFeedforwardNorm,
|
|
1939
725
|
scaleEmbeddings: inf.output.scaleEmbeddings,
|
|
1940
726
|
embeddingScale,
|
|
727
|
+
embeddingNormalization: resolveEmbeddingNormalization(inf.output.embeddingNormalization),
|
|
1941
728
|
logitInputScale,
|
|
729
|
+
logitOutputScale: inf.output.logitOutputScale ?? 1,
|
|
1942
730
|
residualBranchScale,
|
|
1943
731
|
useTiedEmbeddings: inf.output.tieWordEmbeddings,
|
|
1944
732
|
embeddingTranspose: inf.output.embeddingTranspose,
|
|
@@ -1970,6 +758,7 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1970
758
|
attentionOutputGate: inf.attention.attentionOutputGate === true,
|
|
1971
759
|
outputGateType: inf.attention.outputGateType ?? null,
|
|
1972
760
|
queryPreAttnScalar,
|
|
761
|
+
...heterogeneousAttention,
|
|
1973
762
|
layerPipeline: inf.pipeline ?? null,
|
|
1974
763
|
chatTemplateType,
|
|
1975
764
|
chatTemplateEnabled,
|
|
@@ -1985,7 +774,6 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1985
774
|
};
|
|
1986
775
|
}
|
|
1987
776
|
|
|
1988
|
-
|
|
1989
777
|
export function parseModelConfigFromManifest(manifest, runtimeOverrides) {
|
|
1990
778
|
assertSupportedRuntimeModelType(manifest);
|
|
1991
779
|
validateModelOverrides(runtimeOverrides, 'runtime.inference.modelOverrides');
|
|
@@ -2020,12 +808,10 @@ export function parseModelConfigFromManifest(manifest, runtimeOverrides) {
|
|
|
2020
808
|
// Convert to ParsedModelConfig
|
|
2021
809
|
return toParsedConfigFromMerged(merged, manifest);
|
|
2022
810
|
}
|
|
2023
|
-
|
|
2024
811
|
// =============================================================================
|
|
2025
812
|
// Main Entry Point
|
|
2026
813
|
// =============================================================================
|
|
2027
814
|
|
|
2028
|
-
|
|
2029
815
|
export function parseModelConfig(manifest, runtimeOverrides) {
|
|
2030
816
|
// Manifest-first architecture: inference config is required
|
|
2031
817
|
if (!hasManifestInference(manifest)) {
|