doppler-gpu 0.4.15 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -577
- package/LICENSE +21 -201
- package/README.md +165 -213
- package/models/catalog.json +241 -33
- package/package.json +122 -86
- package/src/capsule-runtime.d.ts +23 -0
- package/src/capsule-runtime.js +23 -0
- package/src/capsule.d.ts +8 -0
- package/src/capsule.js +4 -0
- package/src/cli/argument-contract.js +200 -0
- package/src/cli/browser-report-output.js +24 -0
- package/src/cli/cli-model-resolution.js +2 -2
- package/src/cli/cli-output.js +10 -0
- package/src/cli/commands/bundle.js +828 -0
- package/src/cli/doppler-cli.js +101 -1349
- package/src/cli/doppler-serve.js +18 -244
- package/src/cli/output/manifest.js +368 -0
- package/src/cli/runtime-profiles.js +4 -8
- package/src/cli/serve/capsule-handler.d.ts +33 -0
- package/src/cli/serve/capsule-handler.js +105 -0
- package/src/cli/serve/capsule-http.js +102 -0
- package/src/cli/serve/receipt.js +269 -0
- package/src/client/capsule-host.browser.js +1 -0
- package/src/client/capsule-host.d.ts +3 -0
- package/src/client/capsule-host.js +1 -0
- package/src/client/doppler-api.browser.js +42 -0
- package/src/client/doppler-api.d.ts +64 -1
- package/src/client/doppler-api.js +42 -1
- package/src/client/doppler-registry.js +15 -7
- package/src/client/electron/index.d.ts +4 -0
- package/src/client/electron/index.js +13 -0
- package/src/client/electron/ipc-contract.d.ts +9 -0
- package/src/client/electron/ipc-contract.js +56 -0
- package/src/client/electron/release-state.d.ts +53 -0
- package/src/client/electron/release-state.js +425 -0
- package/src/client/electron/renderer-runtime.d.ts +19 -0
- package/src/client/electron/renderer-runtime.js +104 -0
- package/src/client/failure-taxonomy.js +1 -28
- package/src/client/fault-injection.js +0 -8
- package/src/client/inspection.d.ts +122 -0
- package/src/client/inspection.js +343 -0
- package/src/client/model-host/index.d.ts +140 -0
- package/src/client/model-host/index.js +548 -0
- package/src/client/model-host/model-evidence.js +255 -0
- package/src/client/model-host/model-session.d.ts +322 -0
- package/src/client/model-host/model-session.js +593 -0
- package/src/client/provider.d.ts +18 -2
- package/src/client/provider.js +26 -3
- package/src/client/receipt.js +30 -72
- package/src/client/runtime/capsule-acquisition.d.ts +18 -0
- package/src/client/runtime/capsule-acquisition.js +103 -0
- package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
- package/src/client/runtime/capsule-adapter-execution.js +63 -0
- package/src/client/runtime/capsule-artifact-source.js +62 -0
- package/src/client/runtime/capsule-embedding.d.ts +29 -0
- package/src/client/runtime/capsule-embedding.js +76 -0
- package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
- package/src/client/runtime/capsule-forecast-program.js +85 -0
- package/src/client/runtime/capsule-forecast.d.ts +17 -0
- package/src/client/runtime/capsule-forecast.js +31 -0
- package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
- package/src/client/runtime/capsule-operation-adapters.js +68 -0
- package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
- package/src/client/runtime/capsule-operation-executor.js +100 -0
- package/src/client/runtime/capsule-program-adapter.js +151 -0
- package/src/client/runtime/capsule-release-authorization.js +25 -0
- package/src/client/runtime/capsule-rerank.d.ts +50 -0
- package/src/client/runtime/capsule-rerank.js +105 -0
- package/src/client/runtime/capsule-session-execution.js +85 -0
- package/src/client/runtime/command-executor.d.ts +15 -0
- package/src/client/runtime/command-executor.js +128 -0
- package/src/client/runtime/composition-root.d.ts +74 -0
- package/src/client/runtime/composition-root.js +344 -0
- package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
- package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
- package/src/client/runtime/index.d.ts +1 -63
- package/src/client/runtime/index.js +1 -261
- package/src/client/runtime/lora.d.ts +14 -0
- package/src/client/runtime/lora.js +19 -1
- package/src/client/runtime/model-manager.d.ts +0 -4
- package/src/client/runtime/model-manager.js +12 -28
- package/src/client/runtime/model-rerank.js +30 -0
- package/src/client/runtime/model-session.d.ts +1 -174
- package/src/client/runtime/model-source.d.ts +19 -1
- package/src/client/runtime/model-source.js +46 -17
- package/src/client/runtime/node-quickstart-cache.js +1 -1
- package/src/client/runtime/resolution-policy.d.ts +20 -0
- package/src/client/runtime/resolution-policy.js +84 -0
- package/src/client/runtime/resource-binder.d.ts +24 -0
- package/src/client/runtime/resource-binder.js +135 -0
- package/src/client/runtime/scoped-session.d.ts +126 -0
- package/src/client/runtime/scoped-session.js +276 -0
- package/src/client/runtime/session-controller.d.ts +30 -0
- package/src/client/runtime/session-controller.js +170 -0
- package/src/client/runtime/source-runtime.js +1 -1
- package/src/client/runtime/target-selector.d.ts +9 -0
- package/src/client/runtime/target-selector.js +2 -0
- package/src/client/runtime/types.d.ts +1 -1
- package/src/client/runtime/types.js +0 -1
- package/src/client/runtime/verified-capsule-artifact-store.js +72 -0
- package/src/client/wrap-pipeline-handle.js +3 -9
- package/src/config/backward-registry-loader.js +1 -1
- package/src/config/capsule-adapter-policy.js +30 -0
- package/src/config/capsule-adapters.d.ts +21 -0
- package/src/config/capsule-adapters.js +40 -0
- package/src/config/capsule-adapters.json +9 -0
- package/src/config/capsule-loading.d.ts +2 -0
- package/src/config/capsule-loading.js +14 -0
- package/src/config/capsule-loading.schema.json +11 -0
- package/src/config/capsule-operation.d.ts +19 -0
- package/src/config/capsule-operation.js +58 -0
- package/src/config/capsule-operations.json +18 -0
- package/src/config/capsule-release-contract.d.ts +66 -0
- package/src/config/capsule-release-contract.js +197 -0
- package/src/config/capsule-release-events.d.ts +53 -0
- package/src/config/capsule-release-events.js +172 -0
- package/src/config/capsule-sequence-receipt.js +19 -0
- package/src/config/capsule-serve.d.ts +9 -0
- package/src/config/capsule-serve.js +28 -0
- package/src/config/capsule-serve.schema.json +14 -0
- package/src/config/capsule-signature.d.ts +6 -0
- package/src/config/capsule-signature.js +50 -0
- package/src/config/capsule-v2.d.ts +70 -0
- package/src/config/capsule-v2.js +431 -0
- package/src/config/capsule-v3.d.ts +19 -0
- package/src/config/capsule-v3.js +72 -0
- package/src/config/capsule.d.ts +22 -0
- package/src/config/capsule.js +52 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +41 -1
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +1 -1
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +59 -7
- package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
- package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
- package/src/config/dtype-transition-contract.js +40 -0
- package/src/config/embedding-contract.js +26 -0
- package/src/config/embedding-reference.js +85 -0
- package/src/config/evidence/boundary-evidence-policy.json +54 -0
- package/src/config/evidence/source-intake-policy.json +122 -0
- package/src/config/execution-contract-check.d.ts +16 -73
- package/src/config/execution-contract-check.js +7 -429
- package/src/config/forecast-manifest.js +56 -0
- package/src/config/heterogeneous-attention-contract.js +46 -0
- package/src/config/initial-execution-identity.d.ts +58 -0
- package/src/config/initial-execution-identity.js +331 -0
- package/src/config/inspection/observation-policies.json +80 -0
- package/src/config/kernel-path-loader.d.ts +10 -0
- package/src/config/kernel-path-loader.js +23 -8
- package/src/config/kernel-registry-contract.d.ts +25 -0
- package/src/config/kernel-registry-contract.js +46 -0
- package/src/config/kernels/kernel-ref-digests.js +50 -2
- package/src/config/kernels/registry.js +1 -1
- package/src/config/kernels/registry.json +13592 -11377
- package/src/config/lora-layouts.d.ts +10 -0
- package/src/config/lora-layouts.js +17 -0
- package/src/config/lora-layouts.json +9 -0
- package/src/config/manifest-inference-validation.js +575 -0
- package/src/config/merge/execution-patches.js +12 -0
- package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
- package/src/config/merge/value-selection.js +38 -0
- package/src/config/merge.js +43 -11
- package/src/config/model-ir-v2.d.ts +57 -0
- package/src/config/model-ir-v2.js +258 -0
- package/src/config/model-ir.d.ts +49 -0
- package/src/config/model-ir.js +180 -0
- package/src/config/param-validator.js +22 -37
- package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
- package/src/config/platforms/loader.js +2 -1
- package/src/config/production-release-evidence.d.ts +23 -0
- package/src/config/production-release-evidence.js +423 -0
- package/src/config/production-release.js +321 -0
- package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
- package/src/config/required-inference-fields-contract-check.js +2 -2
- package/src/config/rerank-reference.d.ts +28 -0
- package/src/config/rerank-reference.js +89 -0
- package/src/config/revocation-policy.d.ts +65 -0
- package/src/config/revocation-policy.js +290 -0
- package/src/config/revocation-registry.json +11 -0
- package/src/config/revocation-updates.d.ts +68 -0
- package/src/config/revocation-updates.js +440 -0
- package/src/config/runtime/compute/f16-activations.json +1 -0
- package/src/config/runtime/compute/f16-batched.json +1 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/diagnostics/debug-logits.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +1 -3
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/verify/gemma3-verify.json +1 -3
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +1 -3
- package/src/config/runtime/experiments/verify/lfm2-verify.json +1 -3
- package/src/config/runtime/experiments/verify/translategemma-verify.json +1 -3
- package/src/config/runtime/model/gemma2-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline.json +1 -0
- package/src/config/runtime/model/gemma3-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +1 -0
- package/src/config/runtime/optimization-candidates/registry.json +5 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +1 -0
- package/src/config/runtime/profiles/default.json +1 -0
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +1 -3
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +1 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +2 -6
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
- package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
- package/src/config/runtime/profiles/production.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +1 -5
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +1 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +1 -3
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +2 -6
- package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
- package/src/config/runtime/profiles/rerank-stability.json +1 -3
- package/src/config/runtime/profiles/rerank-throughput.json +1 -3
- package/src/config/runtime/profiles/simulation.json +1 -3
- package/src/config/runtime/profiles/throughput.json +1 -3
- package/src/config/runtime/profiles/trace-layers.json +1 -3
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +1 -0
- package/src/config/runtime/profiles/turboquant.json +1 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -3
- package/src/config/runtime/profiles/vector-throughput.json +1 -0
- package/src/config/runtime/profiles/verbose-trace.json +1 -3
- package/src/config/runtime/tiers/gemma4-128gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-24gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-32gb.json +1 -0
- package/src/config/runtime-merge.js +1 -1
- package/src/config/runtime.js +1 -1
- package/src/config/schema/benchmark.schema.d.ts +7 -0
- package/src/config/schema/benchmark.schema.js +5 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/conversion-report.schema.d.ts +3 -0
- package/src/config/schema/conversion-report.schema.js +29 -0
- package/src/config/schema/converter.schema.d.ts +1 -0
- package/src/config/schema/debug.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +5 -5
- package/src/config/schema/emulation.schema.js +1 -1
- package/src/config/schema/execution-v1.schema.d.ts +3 -0
- package/src/config/schema/execution-v1.schema.js +19 -0
- package/src/config/schema/harness.schema.d.ts +13 -0
- package/src/config/schema/harness.schema.js +6 -0
- package/src/config/schema/index.d.ts +8 -16
- package/src/config/schema/index.js +5 -8
- package/src/config/schema/inference.schema.d.ts +21 -2
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -12
- package/src/config/schema/kernel-thresholds.schema.js +1 -13
- package/src/config/schema/kernel-warmup.schema.d.ts +1 -3
- package/src/config/schema/kernel-warmup.schema.js +0 -1
- package/src/config/schema/manifest.schema.d.ts +26 -7
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/program-bundle.schema.d.ts +52 -6
- package/src/config/schema/program-bundle.schema.js +145 -10
- package/src/config/schema/runtime-profile.schema.d.ts +24 -0
- package/src/config/schema/runtime-profile.schema.js +92 -0
- package/src/config/schema/shared-runtime.schema.d.ts +1 -4
- package/src/config/schema/shared-runtime.schema.js +0 -2
- package/src/config/schema/tooling.schema.d.ts +8 -5
- package/src/config/schema/tooling.schema.js +3 -2
- package/src/config/sequence-reference.d.ts +26 -0
- package/src/config/sequence-reference.js +100 -0
- package/src/config/target-plan.d.ts +108 -0
- package/src/config/target-plan.js +411 -0
- package/src/config/training-defaults.js +1 -1
- package/src/config/transforms/capability-transform-resolver.js +4 -105
- package/src/config/transforms/execution-graph/dtype.js +591 -0
- package/src/config/transforms/execution-graph/fusion.js +436 -0
- package/src/config/transforms/execution-graph/validation.js +414 -0
- package/src/config/transforms/execution-graph-transforms.js +13 -1419
- package/src/converter/artifact-identity.js +475 -0
- package/src/converter/conversion-plan.js +1 -1
- package/src/converter/core.js +94 -2096
- package/src/converter/forge-candidate-evaluation.js +271 -0
- package/src/converter/forge-qualification.js +162 -0
- package/src/converter/forge-stages.js +798 -0
- package/src/converter/manifest-builder.js +683 -0
- package/src/converter/quantization-info.js +4 -1
- package/src/converter/quantizer.js +58 -9
- package/src/converter/shard-packer.d.ts +1 -1
- package/src/converter/shard-packer.js +2 -2
- package/src/converter/tensor-transform.js +640 -0
- package/src/converter/tokenizer-contract.js +243 -0
- package/src/converter/tokenizer-manifest.js +243 -0
- package/src/debug/history.js +0 -8
- package/src/debug/index.js +0 -10
- package/src/debug/tensor.d.ts +2 -0
- package/src/debug/tensor.js +21 -2
- package/src/debug/trace.js +0 -7
- package/src/experimental/adapters/adapter-manager.js +2 -0
- package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
- package/src/experimental/adapters/adapter-manifest.js +11 -0
- package/src/experimental/adapters/artifact-contract.d.ts +2 -2
- package/src/experimental/adapters/artifact-contract.js +5 -2
- package/src/experimental/adapters/lora-loader.d.ts +10 -1
- package/src/experimental/adapters/lora-loader.js +110 -48
- package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
- package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
- package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
- package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
- package/src/experimental/browser/browser-converter.js +11 -11
- package/src/experimental/diffusion/index.d.ts +1 -1
- package/src/experimental/diffusion/index.js +1 -1
- package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
- package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
- package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
- package/src/experimental/diffusion/vae.js +596 -0
- package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
- package/src/experimental/distribution/shard-delivery/retry.js +392 -0
- package/src/experimental/distribution/shard-delivery/transport.js +831 -0
- package/src/experimental/distribution/shard-delivery.js +4 -1709
- package/src/experimental/energy/index.d.ts +2 -2
- package/src/experimental/energy/index.js +2 -2
- package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
- package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
- package/src/experimental/logits/cpu-output.d.ts +27 -0
- package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
- package/src/experimental/logits/cpu.d.ts +87 -0
- package/src/experimental/logits/cpu.js +95 -0
- package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
- package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
- package/src/experimental/orchestration/index.d.ts +1 -1
- package/src/experimental/orchestration/index.js +1 -1
- package/src/experimental/training/artifacts.js +2 -2
- package/src/experimental/training/checkpoint.js +2 -2
- package/src/experimental/training/distillation/artifacts.js +1 -1
- package/src/experimental/training/distillation/dataset.js +2 -2
- package/src/experimental/training/distillation/eval.js +1 -1
- package/src/experimental/training/distillation/runtime.js +2 -2
- package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
- package/src/experimental/training/distillation/student-fixture/model.js +353 -0
- package/src/experimental/training/distillation/student-fixture.js +2 -469
- package/src/experimental/training/lora/checkpoint.js +746 -0
- package/src/experimental/training/lora/execution.js +550 -0
- package/src/experimental/training/lora/plan.js +387 -0
- package/src/experimental/training/lora/recovery.js +418 -0
- package/src/experimental/training/lora-pipeline.js +7 -1949
- package/src/experimental/training/operator-artifacts.js +2 -2
- package/src/experimental/training/operator-command.js +4 -4
- package/src/experimental/training/runner/evaluation.js +62 -0
- package/src/experimental/training/runner/lifecycle.js +257 -0
- package/src/experimental/training/runner.js +5 -268
- package/src/experimental/training/suite/execution.js +879 -0
- package/src/experimental/training/suite/plan.js +855 -0
- package/src/experimental/training/suite.js +5 -1659
- package/src/experimental/training/workloads.d.ts +5 -5
- package/src/experimental/training/workloads.js +6 -6
- package/src/experimental/turboquant/codebook.js +470 -0
- package/src/formats/quantization-constants.d.ts +11 -0
- package/src/formats/quantization-constants.js +11 -0
- package/src/formats/rdrr/integrity.js +1 -1
- package/src/formats/rdrr/lowerings.js +0 -29
- package/src/formats/rdrr/merkle.js +1 -1
- package/src/formats/rdrr/tensor-config-validator.js +25 -5
- package/src/formats/rdrr/validation.js +3 -3
- package/src/gpu/command-recorder.d.ts +15 -1
- package/src/gpu/command-recorder.js +63 -2
- package/src/gpu/constant-buffer.js +29 -0
- package/src/gpu/device-state.js +36 -0
- package/src/gpu/device.js +6 -41
- package/src/gpu/kernel-runtime.js +3 -10
- package/src/gpu/kernels/activation-static-qdq.js +1 -1
- package/src/gpu/kernels/append_embedding.wgsl +14 -0
- package/src/gpu/kernels/attention/executors.js +960 -0
- package/src/gpu/kernels/attention/plan.js +472 -0
- package/src/gpu/kernels/attention.d.ts +4 -0
- package/src/gpu/kernels/attention.js +145 -1545
- package/src/gpu/kernels/attention_relative.wgsl +59 -0
- package/src/gpu/kernels/backward/adam.js +2 -1
- package/src/gpu/kernels/backward/attention_backward.js +1 -1
- package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
- package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
- package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
- package/src/gpu/kernels/backward/embed_backward.js +2 -1
- package/src/gpu/kernels/backward/executor.d.ts +34 -0
- package/src/gpu/kernels/backward/executor.js +158 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
- package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/gelu_backward.js +1 -1
- package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
- package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
- package/src/gpu/kernels/backward/matmul.js +133 -0
- package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
- package/src/gpu/kernels/backward/matmul_backward.js +1 -1
- package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
- package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
- package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/rope_backward.js +2 -1
- package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/scale_backward.js +2 -1
- package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/silu_backward.js +1 -1
- package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/softmax_backward.js +1 -1
- package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
- package/src/gpu/kernels/backward/utils.js +2 -392
- package/src/gpu/kernels/bind-group-validation.js +16 -0
- package/src/gpu/kernels/cast.js +2 -1
- package/src/gpu/kernels/check-finiteness.js +13 -49
- package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
- package/src/gpu/kernels/check-stop.js +14 -33
- package/src/gpu/kernels/check_finiteness.wgsl +33 -0
- package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
- package/src/gpu/kernels/check_stop.wgsl +20 -0
- package/src/gpu/kernels/clamp.js +1 -1
- package/src/gpu/kernels/conv2d.js +1 -1
- package/src/gpu/kernels/cross_entropy_loss.js +1 -1
- package/src/gpu/kernels/depthwise_conv1d.js +1 -1
- package/src/gpu/kernels/depthwise_conv2d.js +1 -1
- package/src/gpu/kernels/dequant.js +7 -2
- package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
- package/src/gpu/kernels/dispatch.js +12 -1
- package/src/gpu/kernels/embedding-pool.js +72 -0
- package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
- package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
- package/src/gpu/kernels/energy.js +2 -1
- package/src/gpu/kernels/forecast_output.wgsl +17 -0
- package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
- package/src/gpu/kernels/fused_ffn.js +10 -220
- package/src/gpu/kernels/fused_matmul_residual.js +2 -1
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
- package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
- package/src/gpu/kernels/gated-short-conv.js +11 -69
- package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
- package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
- package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
- package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
- package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
- package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
- package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
- package/src/gpu/kernels/gather.d.ts +1 -1
- package/src/gpu/kernels/gather.js +30 -6
- package/src/gpu/kernels/gather_bf16.wgsl +69 -0
- package/src/gpu/kernels/gelu.js +1 -1
- package/src/gpu/kernels/gemma4-route-expert.js +1 -1
- package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
- package/src/gpu/kernels/groupnorm.js +1 -1
- package/src/gpu/kernels/index.d.ts +38 -7
- package/src/gpu/kernels/index.js +18 -6
- package/src/gpu/kernels/instance_stats.wgsl +26 -0
- package/src/gpu/kernels/kernel-base.js +1 -1
- package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
- package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
- package/src/gpu/kernels/kernel-configs.d.ts +10 -61
- package/src/gpu/kernels/kernel-configs.js +26 -54
- package/src/gpu/kernels/kernel-execution.js +141 -0
- package/src/gpu/kernels/kernel-prewarm.d.ts +3 -0
- package/src/gpu/kernels/kernel-prewarm.js +60 -0
- package/src/gpu/kernels/kv-cache-write.js +4 -4
- package/src/gpu/kernels/kv-quantize.js +4 -1
- package/src/gpu/kernels/l2-normalize.js +36 -0
- package/src/gpu/kernels/l2_normalize.wgsl +53 -0
- package/src/gpu/kernels/layernorm.d.ts +4 -4
- package/src/gpu/kernels/layernorm.js +44 -7
- package/src/gpu/kernels/linear-attention-core.js +29 -449
- package/src/gpu/kernels/linear_attention.js +1 -1
- package/src/gpu/kernels/lm-head-argmax.js +2 -2
- package/src/gpu/kernels/lm-head-select-logits.js +2 -2
- package/src/gpu/kernels/logit-finalize.js +103 -0
- package/src/gpu/kernels/logit-merge.js +13 -125
- package/src/gpu/kernels/logit-scatter.js +40 -0
- package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
- package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
- package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
- package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
- package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
- package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
- package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
- package/src/gpu/kernels/matmul-dispatch.js +6 -1
- package/src/gpu/kernels/matmul-selection.js +3 -2
- package/src/gpu/kernels/matmul.js +4 -15
- package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
- package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
- package/src/gpu/kernels/modulate.js +1 -1
- package/src/gpu/kernels/moe.js +3 -1
- package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
- package/src/gpu/kernels/patch_context.wgsl +29 -0
- package/src/gpu/kernels/pipeline-cache.js +3 -1
- package/src/gpu/kernels/pixel_shuffle.js +1 -1
- package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
- package/src/gpu/kernels/relu.js +1 -1
- package/src/gpu/kernels/repeat_channels.js +1 -1
- package/src/gpu/kernels/residual.js +2 -2
- package/src/gpu/kernels/rmsnorm-pair.js +1 -1
- package/src/gpu/kernels/rmsnorm-qk.js +1 -1
- package/src/gpu/kernels/rmsnorm-stats.js +14 -113
- package/src/gpu/kernels/rmsnorm.js +1 -1
- package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
- package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
- package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
- package/src/gpu/kernels/rope-precompute.js +160 -0
- package/src/gpu/kernels/rope-qk.js +1 -1
- package/src/gpu/kernels/rope.js +2 -2
- package/src/gpu/kernels/rope_precompute.wgsl +74 -0
- package/src/gpu/kernels/sample.js +5 -1
- package/src/gpu/kernels/scale.js +1 -1
- package/src/gpu/kernels/shader-cache.js +8 -4
- package/src/gpu/kernels/shader-source-scope.js +68 -0
- package/src/gpu/kernels/silu.js +2 -1
- package/src/gpu/kernels/soft-embedding.js +3 -1
- package/src/gpu/kernels/softmax.js +4 -2
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
- package/src/gpu/kernels/split_qg.js +1 -1
- package/src/gpu/kernels/split_qkv.d.ts +2 -0
- package/src/gpu/kernels/split_qkv.js +18 -3
- package/src/gpu/kernels/split_qkv.wgsl +5 -2
- package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
- package/src/gpu/kernels/submit_probe.wgsl +6 -0
- package/src/gpu/kernels/transpose.js +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +4 -535
- package/src/gpu/kernels/uniform-utils.js +7 -2
- package/src/gpu/kernels/upsample2d.js +1 -1
- package/src/gpu/kernels/utils.js +9 -332
- package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
- package/src/gpu/kernels/vision-average-pool.js +68 -0
- package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
- package/src/gpu/kernels/vision-patch-embed.js +117 -0
- package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
- package/src/gpu/kernels/vision-position-embedding.js +79 -0
- package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
- package/src/gpu/kernels/vision-rope-2d.js +73 -0
- package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
- package/src/gpu/kernels/vision-spatial-merge.js +100 -0
- package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
- package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
- package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
- package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
- package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
- package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
- package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
- package/src/gpu/readback-buffer.js +20 -0
- package/src/gpu/submit-probe.js +2 -26
- package/src/gpu/weight-buffer.d.ts +1 -0
- package/src/gpu/weight-buffer.js +7 -0
- package/src/index.d.ts +23 -1
- package/src/index.js +8 -1
- package/src/inference/browser-harness/benchmark-execution.js +872 -0
- package/src/inference/browser-harness/benchmark-rerank.js +284 -0
- package/src/inference/browser-harness/decode-diagnostics.js +241 -0
- package/src/inference/browser-harness/diffusion-performance.js +102 -0
- package/src/inference/browser-harness/inference-execution.js +543 -0
- package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +18 -145
- package/src/inference/browser-harness/model-resolution.js +137 -0
- package/src/inference/browser-harness/report.js +338 -0
- package/src/inference/browser-harness/request.js +263 -0
- package/src/inference/browser-harness/runtime-config.d.ts +25 -0
- package/src/inference/browser-harness/runtime-config.js +255 -0
- package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
- package/src/inference/browser-harness/runtime-isolation.js +45 -0
- package/src/inference/browser-harness/runtime-report.d.ts +26 -0
- package/src/inference/browser-harness/runtime-report.js +117 -0
- package/src/inference/browser-harness/suite-summary.js +39 -0
- package/src/inference/browser-harness/text-evidence.js +694 -0
- package/src/inference/browser-harness/text-execution.js +757 -0
- package/src/inference/browser-harness/text-input.js +725 -0
- package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
- package/src/inference/browser-harness-contract-helpers.js +1 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
- package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
- package/src/inference/browser-harness-runtime-helpers.js +3 -408
- package/src/inference/browser-harness-text-helpers.js +31 -2196
- package/src/inference/browser-harness.d.ts +19 -2
- package/src/inference/browser-harness.js +46 -1873
- package/src/inference/kv-cache/base.js +67 -381
- package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
- package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
- package/src/inference/kv-cache/quantized.d.ts +2 -2
- package/src/inference/kv-cache/quantized.js +4 -8
- package/src/inference/kv-cache/update.js +309 -0
- package/src/inference/model-checkpoint-evidence.d.ts +75 -0
- package/src/inference/model-checkpoint-evidence.js +264 -0
- package/src/inference/moe-router.js +36 -216
- package/src/inference/multi-model-network.js +5 -341
- package/src/inference/observation-context.d.ts +22 -0
- package/src/inference/observation-context.js +41 -0
- package/src/inference/pipelines/audio/index.js +1 -20
- package/src/inference/pipelines/audio/mel.js +0 -38
- package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
- package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
- package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
- package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
- package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
- package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
- package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
- package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
- package/src/inference/pipelines/factory.js +14 -3
- package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +0 -6
- package/src/inference/pipelines/text/abort-contract.js +22 -0
- package/src/inference/pipelines/text/attention/attn-config.js +1 -16
- package/src/inference/pipelines/text/attention/executor-immediate.js +42 -0
- package/src/inference/pipelines/text/attention/executor-recorded.js +29 -0
- package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
- package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
- package/src/inference/pipelines/text/attention/index.js +7 -0
- package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
- package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
- package/src/inference/pipelines/text/attention/interpreter.js +984 -0
- package/src/inference/pipelines/text/attention/plan.d.ts +37 -0
- package/src/inference/pipelines/text/attention/plan.js +345 -0
- package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
- package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
- package/src/inference/pipelines/text/attention/projections.js +20 -537
- package/src/inference/pipelines/text/attention/query-transform.js +11 -0
- package/src/inference/pipelines/text/attention/receipt.js +60 -0
- package/src/inference/pipelines/text/attention/record.js +1 -1277
- package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
- package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
- package/src/inference/pipelines/text/attention/run.js +1 -1385
- package/src/inference/pipelines/text/attention/types.js +3 -31
- package/src/inference/pipelines/text/chat-format.d.ts +3 -0
- package/src/inference/pipelines/text/chat-format.js +42 -0
- package/src/inference/pipelines/text/config/normalization.js +729 -0
- package/src/inference/pipelines/text/config/validation.js +9 -0
- package/src/inference/pipelines/text/config.d.ts +12 -1
- package/src/inference/pipelines/text/config.js +14 -1228
- package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
- package/src/inference/pipelines/text/debug-utils/index.js +3 -8
- package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
- package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
- package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +1 -68
- package/src/inference/pipelines/text/embed.js +178 -241
- package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
- package/src/inference/pipelines/text/embedding-contract.js +21 -0
- package/src/inference/pipelines/text/embedding-extraction.js +225 -0
- package/src/inference/pipelines/text/embedding-normalization.js +39 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
- package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
- package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
- package/src/inference/pipelines/text/execution-v1.js +7 -283
- package/src/inference/pipelines/text/execution.js +517 -0
- package/src/inference/pipelines/text/ffn/dense-executor.js +718 -0
- package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
- package/src/inference/pipelines/text/ffn/dense.js +4 -968
- package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
- package/src/inference/pipelines/text/ffn/standard.js +2 -1
- package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
- package/src/inference/pipelines/text/generator/decode-runtime.js +991 -0
- package/src/inference/pipelines/text/generator/decode.js +952 -0
- package/src/inference/pipelines/text/generator/diffusion.js +607 -0
- package/src/inference/pipelines/text/generator/logits-config.js +63 -0
- package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
- package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
- package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
- package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
- package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
- package/src/inference/pipelines/text/generator/recovery.js +310 -0
- package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +44 -110
- package/src/inference/pipelines/text/generator/text.js +448 -0
- package/src/inference/pipelines/text/generator/training.js +752 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +5 -2
- package/src/inference/pipelines/text/generator-logits-observation.js +45 -0
- package/src/inference/pipelines/text/generator-runtime.js +5 -135
- package/src/inference/pipelines/text/generator-steps.js +37 -962
- package/src/inference/pipelines/text/generator.js +165 -3539
- package/src/inference/pipelines/text/image-transcription.js +304 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
- package/src/inference/pipelines/text/init-rope.js +37 -189
- package/src/inference/pipelines/text/init.d.ts +11 -3
- package/src/inference/pipelines/text/init.js +6 -216
- package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
- package/src/inference/pipelines/text/layer-input-execution.js +435 -0
- package/src/inference/pipelines/text/layer-plan-gpu.js +15 -6
- package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
- package/src/inference/pipelines/text/layer-plan.js +45 -3
- package/src/inference/pipelines/text/layer.js +54 -522
- package/src/inference/pipelines/text/lifecycle.js +839 -0
- package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
- package/src/inference/pipelines/text/linear-attention.js +7 -537
- package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
- package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
- package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
- package/src/inference/pipelines/text/logits/cpu.js +8 -95
- package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
- package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
- package/src/inference/pipelines/text/logits/gpu.js +116 -579
- package/src/inference/pipelines/text/logits/index.d.ts +2 -1
- package/src/inference/pipelines/text/logits/index.js +62 -102
- package/src/inference/pipelines/text/logits/output-transform.js +47 -0
- package/src/inference/pipelines/text/logits/plan.js +184 -0
- package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
- package/src/inference/pipelines/text/logits/readback.js +9 -0
- package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/lora-apply.js +12 -4
- package/src/inference/pipelines/text/lora-types.d.ts +12 -0
- package/src/inference/pipelines/text/modality-token-contract.js +115 -0
- package/src/inference/pipelines/text/model-load.d.ts +0 -1
- package/src/inference/pipelines/text/model-load.js +2 -31
- package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
- package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
- package/src/inference/pipelines/text/moe/plan.js +47 -0
- package/src/inference/pipelines/text/moe-cpu.js +9 -135
- package/src/inference/pipelines/text/moe-gpu.js +21 -545
- package/src/inference/pipelines/text/normalization-contract.js +17 -0
- package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
- package/src/inference/pipelines/text/per-layer/plan.js +362 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
- package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
- package/src/inference/pipelines/text/probes.js +30 -0
- package/src/inference/pipelines/text/resolved-runtime-session.d.ts +56 -0
- package/src/inference/pipelines/text/resolved-runtime-session.js +175 -0
- package/src/inference/pipelines/text/state.d.ts +14 -0
- package/src/inference/pipelines/text/state.js +4 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
- package/src/inference/pipelines/text/types.d.ts +10 -0
- package/src/inference/pipelines/text/weight-loading-config.js +194 -0
- package/src/inference/pipelines/text.d.ts +2 -2
- package/src/inference/pipelines/text.js +48 -1620
- package/src/inference/pipelines/video/frame-sampler.js +0 -13
- package/src/inference/pipelines/video/index.js +0 -21
- package/src/inference/pipelines/vision/attention.js +59 -0
- package/src/inference/pipelines/vision/encoder.js +254 -369
- package/src/inference/pipelines/vision/gemma4.js +204 -318
- package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
- package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
- package/src/inference/pipelines/vision/glmocr.js +557 -0
- package/src/inference/pipelines/vision/image-preprocess.js +0 -27
- package/src/inference/pipelines/vision/index.js +8 -27
- package/src/inference/pipelines/vision/ops.js +11 -33
- package/src/inference/pipelines/vision/patch-embed.js +51 -145
- package/src/inference/pipelines/vision/spatial-merge.js +78 -0
- package/src/inference/pipelines/vision/weight-loading.js +79 -0
- package/src/inference/refactor-receipt.d.ts +49 -0
- package/src/inference/refactor-receipt.js +188 -0
- package/src/inference/rerank.js +182 -0
- package/src/inference/resource-scope.d.ts +33 -0
- package/src/inference/resource-scope.js +168 -0
- package/src/inference/test-harness.d.ts +3 -0
- package/src/inference/test-harness.js +1 -0
- package/src/inference/tokenizer.js +1 -1
- package/src/inference/tokenizers/behavior-flags.js +6 -3
- package/src/inference/tokenizers/bundled/decoders.js +133 -0
- package/src/inference/tokenizers/bundled/registry.js +216 -0
- package/src/inference/tokenizers/bundled.js +23 -353
- package/src/loader/doppler-loader.d.ts +4 -0
- package/src/loader/doppler-loader.js +119 -930
- package/src/loader/embedding-limit-preflight.js +11 -4
- package/src/loader/embedding-loader.js +1 -3
- package/src/loader/gpu-tensor-materialization.js +20 -0
- package/src/loader/layer-loader.js +15 -119
- package/src/loader/layer-tensor-aliases.js +99 -0
- package/src/loader/layer-weight-shape.js +29 -0
- package/src/loader/load-timing.js +64 -0
- package/src/loader/loader-types.d.ts +3 -0
- package/src/loader/model-executor.js +819 -0
- package/src/loader/model-load-validation.js +15 -0
- package/src/loader/quantization-constants.d.ts +11 -29
- package/src/loader/quantization-constants.js +1 -2
- package/src/loader/shard-resolver.js +0 -14
- package/src/loader/tensor-load-error.js +14 -0
- package/src/loader/tensor-shard-indices.js +8 -0
- package/src/loader/tensors/bf16-loader.js +125 -0
- package/src/loader/tensors/functional-descriptor-loader.js +3 -375
- package/src/loader/tensors/load-executor.js +577 -0
- package/src/loader/tensors/source-transform/executor.js +123 -0
- package/src/loader/tensors/source-transform/plan.js +107 -0
- package/src/loader/tensors/source-transform.js +15 -264
- package/src/loader/tensors/tensor-loader.d.ts +4 -4
- package/src/loader/tensors/tensor-loader.js +7 -653
- package/src/memory/buffer-pool.js +7 -29
- package/src/rules/capability-transform-resolver.js +80 -0
- package/src/rules/execution-contract.d.ts +73 -0
- package/src/rules/execution-contract.js +429 -0
- package/src/rules/execution-rules-contract-check.js +1 -1
- package/src/rules/generated/rule-bundle.json +4305 -0
- package/src/rules/inference/capability-transforms.rules.json +3 -3
- package/src/rules/layer-pattern-contract-check.js +1 -1
- package/src/rules/rule-registry.js +70 -57
- package/src/storage/artifact-storage-context.js +2 -2
- package/src/storage/backend-selection.js +43 -0
- package/src/storage/distribution-transport.js +5 -0
- package/src/storage/download/integrity.d.ts +33 -0
- package/src/storage/download/integrity.js +130 -0
- package/src/storage/download/resume-inspection.d.ts +22 -0
- package/src/storage/download/resume-inspection.js +134 -0
- package/src/storage/download/retry.js +57 -0
- package/src/storage/download/state.js +131 -0
- package/src/storage/download/transport.js +99 -0
- package/src/storage/downloader.js +74 -453
- package/src/storage/model-cache.d.ts +54 -0
- package/src/storage/model-cache.js +590 -0
- package/src/storage/shard-manager.d.ts +5 -0
- package/src/storage/shard-manager.js +50 -369
- package/src/storage/shards/index.js +60 -0
- package/src/storage/shards/integrity.js +225 -0
- package/src/storage/shards/lifecycle.js +22 -0
- package/src/storage/source-artifact-store.js +14 -4
- package/src/tooling/boundary-evidence.d.ts +51 -0
- package/src/tooling/boundary-evidence.js +408 -0
- package/src/tooling/browser-command-runner.js +17 -10
- package/src/tooling/capsule-v2.d.ts +11 -0
- package/src/tooling/capsule-v2.js +58 -0
- package/src/tooling/capsule.js +20 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.js +163 -5
- package/src/tooling/command-api.d.ts +42 -3
- package/src/tooling/command-api.js +7 -2
- package/src/tooling/command-context.d.ts +25 -0
- package/src/tooling/command-context.js +53 -0
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
- package/src/tooling/command-runner.html +49 -1
- package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +22 -5
- package/src/tooling/command-validation.js +1 -24
- package/src/tooling/conversion-run-timing.js +24 -0
- package/src/tooling/demo-receipts.d.ts +56 -0
- package/src/tooling/demo-receipts.js +113 -0
- package/src/tooling/diagnose-runner.js +13 -11
- package/src/tooling/distribution-shard-transport.js +11 -0
- package/src/tooling/execution-cost-ledger.d.ts +95 -0
- package/src/tooling/execution-cost-ledger.js +381 -0
- package/src/tooling/hf-registry/catalog-io.js +63 -0
- package/src/tooling/litert/package.js +863 -0
- package/src/tooling/litert-package-runtime.js +3 -849
- package/src/tooling/model-capsule-forge.js +478 -0
- package/src/tooling/node-browser/execution.d.ts +40 -0
- package/src/tooling/node-browser/execution.js +308 -0
- package/src/tooling/node-browser/static-server.js +188 -0
- package/src/tooling/node-browser/transport.d.ts +80 -0
- package/src/tooling/node-browser/transport.js +622 -0
- package/src/tooling/node-browser-command-runner.d.ts +18 -78
- package/src/tooling/node-browser-command-runner.js +10 -1112
- package/src/tooling/node-capsule-artifact-store.js +44 -0
- package/src/tooling/node-command-execution.js +237 -0
- package/src/tooling/node-command-runner.js +26 -205
- package/src/tooling/node-converter/input.js +521 -0
- package/src/tooling/node-converter/output.js +244 -0
- package/src/tooling/node-converter-chunk-batches.js +21 -0
- package/src/tooling/node-converter.js +7 -720
- package/src/tooling/node-source-runtime.js +16 -276
- package/src/tooling/node-webgpu-provider.v1.json +33 -0
- package/src/tooling/node-webgpu.js +178 -453
- package/src/tooling/opfs-cache.d.ts +15 -48
- package/src/tooling/opfs-cache.js +8 -510
- package/src/tooling/precision-replay-math.d.ts +42 -0
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/production-release.d.ts +19 -0
- package/src/tooling/production-release.js +647 -0
- package/src/tooling/program-bundle/embedding-reference.js +19 -0
- package/src/tooling/program-bundle/materialize.js +324 -0
- package/src/tooling/program-bundle/rerank-reference.js +22 -0
- package/src/tooling/program-bundle/sequence-reference.js +42 -0
- package/src/tooling/program-bundle/validation.js +31 -0
- package/src/tooling/program-bundle/wgsl-closure.js +254 -0
- package/src/tooling/program-bundle-parity.d.ts +14 -4
- package/src/tooling/program-bundle-parity.js +267 -106
- package/src/tooling/program-bundle.d.ts +8 -52
- package/src/tooling/program-bundle.js +186 -730
- package/src/tooling/provider-v1-contract.d.ts +61 -0
- package/src/tooling/provider-v1-contract.js +192 -0
- package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
- package/src/tooling/registered-variant-calibration-job.js +210 -0
- package/src/tooling/registered-variant-calibration.d.ts +184 -0
- package/src/tooling/registered-variant-calibration.js +336 -0
- package/src/tooling/runtime-input-composition.js +105 -53
- package/src/tooling/runtime-intent.js +41 -0
- package/src/tooling/runtime-optimization/candidates.js +664 -0
- package/src/tooling/runtime-optimization-campaign.js +166 -0
- package/src/tooling/runtime-optimization-index.d.ts +6 -0
- package/src/tooling/runtime-optimization-index.js +87 -0
- package/src/tooling/runtime-optimization.d.ts +91 -6
- package/src/tooling/runtime-optimization.js +290 -414
- package/src/tooling/source-intake.d.ts +49 -0
- package/src/tooling/source-intake.js +431 -0
- package/src/tooling/source-package-profiles.js +1 -1
- package/src/tooling/source-runtime/bundle-materialize.js +298 -0
- package/src/tooling/source-runtime/bundle-plan.js +121 -0
- package/src/tooling/source-runtime/node-loader.js +258 -0
- package/src/tooling/source-runtime-bundle.js +26 -411
- package/src/tooling/source-runtime-converter-config.js +1 -1
- package/src/tooling-experimental-exports.js +0 -5
- package/src/tooling-exports/evidence.d.ts +45 -0
- package/src/tooling-exports/evidence.js +45 -0
- package/src/tooling-exports/runtime.d.ts +7 -0
- package/src/tooling-exports/runtime.js +7 -0
- package/src/tooling-exports/storage.d.ts +5 -0
- package/src/tooling-exports/storage.js +4 -0
- package/src/tooling-exports.d.ts +14 -0
- package/src/tooling-exports.js +8 -3
- package/src/tooling-exports.shared.d.ts +46 -0
- package/src/tooling-exports.shared.js +37 -0
- package/src/training.d.ts +6 -6
- package/src/training.js +4 -4
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +512 -0
- package/tests/kernels/browser/test-page.js +169 -14
- package/tests/kernels/harness/tolerance.js +1 -0
- package/tests/kernels/reference/index.js +13 -1
- package/tests/kernels/reference/vision.js +160 -0
- package/src/client/runtime/model-session.js +0 -432
- package/src/config/schema/tuner.schema.d.ts +0 -39
- package/src/config/schema/tuner.schema.js +0 -13
- package/src/converter/quantizer.d.ts +0 -121
- package/src/gpu/kernel-tuner/benchmarks.js +0 -902
- package/src/gpu/kernel-tuner/cache.js +0 -145
- package/src/gpu/kernel-tuner/index.js +0 -38
- package/src/gpu/kernel-tuner/tuner.js +0 -247
- package/src/gpu/kernel-tuner.js +0 -12
- package/src/gpu/kernels/backward/utils.d.ts +0 -90
- package/src/gpu/kernels/kernel-tuning.d.ts +0 -42
- package/src/gpu/kernels/kernel-tuning.js +0 -158
- package/src/gpu/kernels/uniform-utils.d.ts +0 -66
- package/src/gpu/kernels/utils.d.ts +0 -149
- package/src/gpu/profiler.js +0 -382
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +0 -37
- package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
- package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
- package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
- package/src/inference/pipelines/text/moe-helpers.js +0 -21
- package/src/rules/converter/execution.rules.json +0 -20
- package/src/rules/converter/tensor-roles.rules.json +0 -13
- package/src/rules/converter/tokenizer.rules.json +0 -12
- package/src/rules/inference/attention.rules.json +0 -69
- package/src/rules/inference/config.rules.json +0 -82
- package/src/rules/inference/dtype.rules.json +0 -113
- package/src/rules/inference/execution.rules.json +0 -195
- package/src/rules/inference/ffn.rules.json +0 -123
- package/src/rules/inference/layer-pattern.rules.json +0 -16
- package/src/rules/inference/layer.rules.json +0 -7
- package/src/rules/inference/moe.rules.json +0 -145
- package/src/rules/kernels/attention.rules.json +0 -68
- package/src/rules/kernels/conv2d.rules.json +0 -6
- package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
- package/src/rules/kernels/dequant.rules.json +0 -58
- package/src/rules/kernels/energy.rules.json +0 -22
- package/src/rules/kernels/fused-ffn.rules.json +0 -15
- package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
- package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
- package/src/rules/kernels/gather.rules.json +0 -93
- package/src/rules/kernels/gelu.rules.json +0 -11
- package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
- package/src/rules/kernels/groupnorm.rules.json +0 -10
- package/src/rules/kernels/kv_quantize.rules.json +0 -9
- package/src/rules/kernels/layernorm.rules.json +0 -6
- package/src/rules/kernels/linear-attention.rules.json +0 -6
- package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
- package/src/rules/kernels/matmul.rules.json +0 -95
- package/src/rules/kernels/modulate.rules.json +0 -6
- package/src/rules/kernels/moe.rules.gptoss.json +0 -105
- package/src/rules/kernels/moe.rules.json +0 -14
- package/src/rules/kernels/moe.rules.mixtral.json +0 -75
- package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
- package/src/rules/kernels/relu.rules.json +0 -6
- package/src/rules/kernels/rep-penalty.rules.json +0 -6
- package/src/rules/kernels/repeat-channels.rules.json +0 -6
- package/src/rules/kernels/residual.rules.json +0 -12
- package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
- package/src/rules/kernels/rmsnorm.rules.json +0 -46
- package/src/rules/kernels/rope-qk.rules.json +0 -6
- package/src/rules/kernels/rope.rules.json +0 -6
- package/src/rules/kernels/sample.rules.json +0 -6
- package/src/rules/kernels/scale.rules.json +0 -8
- package/src/rules/kernels/silu.rules.json +0 -22
- package/src/rules/kernels/softmax.rules.json +0 -25
- package/src/rules/kernels/split-qg.rules.json +0 -6
- package/src/rules/kernels/split-qkv.rules.json +0 -6
- package/src/rules/kernels/upsample2d.rules.json +0 -6
- package/src/rules/loader/tensor-loader.rules.json +0 -42
- package/src/rules/loader/weights.rules.json +0 -42
- package/src/rules/tooling/command-runtime.rules.json +0 -65
- package/src/tooling/hf-registry-utils.js +0 -395
- /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
- /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
- /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
- /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
- /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
- /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
- /package/src/{utils → formats}/array-buffer.js +0 -0
- /package/src/{utils → formats}/canonical-hash.js +0 -0
- /package/src/{utils → formats}/clone-json.js +0 -0
- /package/src/{utils → formats}/encode-utf8.js +0 -0
- /package/src/{utils → formats}/load-json.js +0 -0
- /package/src/{utils → formats}/plain-object.js +0 -0
- /package/src/{utils → formats}/sha256.js +0 -0
- /package/src/{utils → formats}/stable-sort-object.js +0 -0
- /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
- /package/src/{utils → storage}/hf-resolve-url.js +0 -0
- /package/src/{utils → storage}/runtime-env.js +0 -0
package/src/converter/core.js
CHANGED
|
@@ -25,7 +25,7 @@ import {
|
|
|
25
25
|
} from '../config/index.js';
|
|
26
26
|
import { buildExecutionContractArtifact } from '../config/execution-contract-check.js';
|
|
27
27
|
import { buildManifestRequiredInferenceFieldsArtifact } from '../config/required-inference-fields-contract-check.js';
|
|
28
|
-
import { resolveEosTokenId } from './tokenizer-
|
|
28
|
+
import { resolveEosTokenId } from './tokenizer-special-tokens.js';
|
|
29
29
|
import { inferBundledTokenizerBehaviorFlags } from '../inference/tokenizers/behavior-flags.js';
|
|
30
30
|
import {
|
|
31
31
|
normalizeQ4KLayout,
|
|
@@ -40,244 +40,32 @@ import {
|
|
|
40
40
|
quantizeToQ4KMColumnWise,
|
|
41
41
|
quantizeToInt4PerRowSymmetric,
|
|
42
42
|
} from './quantizer.js';
|
|
43
|
-
import { cloneJsonValue } from '../
|
|
43
|
+
import { cloneJsonValue } from '../formats/clone-json.js';
|
|
44
|
+
import { SOURCE_PACKED_QUANT_DTYPES, isGemma4PerLayerEmbedTensor, normalizeModulesToNotConvert, normalizeStorageQuant, resolveQuantizeEmbeddings, transformTensorBytes } from './tensor-transform.js';
|
|
45
|
+
import { buildArtifactIdentity, extractArchitecture, getNestedTextConfig, sanitizeModelId } from './artifact-identity.js';
|
|
46
|
+
import { RDRR_VERSION, createManifest, modelHasMoETensors } from './manifest-builder.js';
|
|
47
|
+
import { normalizeCompressedTensorsW4A16, normalizeTensorName } from './tokenizer-contract.js';
|
|
48
|
+
export { RDRR_VERSION, buildBundledTokenizer, buildSentencepieceTokenizer, createManifest, inferEmbeddingOutputConfig, resolveBundledTokenizerVocabSize, resolveConvertedAt, resolveManifestMoEConfig, resolveManifestMultimodalConfig } from './manifest-builder.js';
|
|
49
|
+
export { extractArchitecture, sanitizeModelId } from './artifact-identity.js';
|
|
50
|
+
export { isGemma4PerLayerEmbedTensor, normalizeStorageQuant, resolveTensorTargetQuant, shouldQuantize, transformTensorBytes } from './tensor-transform.js';
|
|
44
51
|
|
|
45
52
|
// ============================================================================
|
|
46
53
|
// Re-exports for Backward Compatibility
|
|
47
54
|
// ============================================================================
|
|
48
55
|
|
|
49
|
-
|
|
50
56
|
export const ConvertStage = SchemaConversionStage;
|
|
51
57
|
|
|
52
58
|
// Re-export constants
|
|
53
59
|
export const SHARD_SIZE = SCHEMA_SHARD_SIZE;
|
|
54
|
-
export const RDRR_VERSION = SCHEMA_RDRR_VERSION;
|
|
55
60
|
|
|
56
61
|
// ============================================================================
|
|
57
62
|
// Embedding Output Inference
|
|
58
63
|
// ============================================================================
|
|
59
64
|
|
|
60
|
-
const EMBEDDING_TENSOR_NAMES = [
|
|
61
|
-
'language_model.model.embed_tokens.weight',
|
|
62
|
-
'model.embed_tokens.weight',
|
|
63
|
-
'embed_tokens.weight',
|
|
64
|
-
'token_embd.weight',
|
|
65
|
-
'wte.weight',
|
|
66
|
-
'transformer.wte.weight',
|
|
67
|
-
];
|
|
68
|
-
|
|
69
|
-
export function inferEmbeddingOutputConfig(tensorLocations) {
|
|
70
|
-
// Normalize Map input to a plain object so the rest of the function
|
|
71
|
-
// handles a single type consistently.
|
|
72
|
-
const normalized = tensorLocations instanceof Map
|
|
73
|
-
? Object.fromEntries(tensorLocations)
|
|
74
|
-
: tensorLocations ?? {};
|
|
75
|
-
|
|
76
|
-
const getLocation = (name) => normalized[name];
|
|
77
|
-
|
|
78
|
-
const entries = Object.entries(normalized);
|
|
79
|
-
for (const [_name, loc] of entries) {
|
|
80
|
-
if (loc?.role === 'embedding' && loc.shape?.length === 2) {
|
|
81
|
-
const [dim0, dim1] = loc.shape;
|
|
82
|
-
const isGGUFLayout = dim0 < dim1;
|
|
83
|
-
return {
|
|
84
|
-
embeddingTranspose: isGGUFLayout,
|
|
85
|
-
embeddingVocabSize: isGGUFLayout ? dim1 : dim0,
|
|
86
|
-
};
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
for (const name of EMBEDDING_TENSOR_NAMES) {
|
|
91
|
-
const loc = getLocation(name);
|
|
92
|
-
if (loc?.shape && loc.shape.length === 2) {
|
|
93
|
-
const [dim0, dim1] = loc.shape;
|
|
94
|
-
const isGGUFLayout = dim0 < dim1;
|
|
95
|
-
return {
|
|
96
|
-
embeddingTranspose: isGGUFLayout,
|
|
97
|
-
embeddingVocabSize: isGGUFLayout ? dim1 : dim0,
|
|
98
|
-
};
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
return null;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
65
|
// ============================================================================
|
|
106
66
|
// Pure Functions (no I/O, no platform dependencies)
|
|
107
67
|
// ============================================================================
|
|
108
68
|
|
|
109
|
-
function resolveTokenizerId(value) {
|
|
110
|
-
if (typeof value === 'number') return value;
|
|
111
|
-
return null;
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
function resolveTokenizerIds(value) {
|
|
115
|
-
if (Array.isArray(value) && value.every((id) => typeof id === 'number')) {
|
|
116
|
-
return value;
|
|
117
|
-
}
|
|
118
|
-
if (typeof value === 'number') return [value];
|
|
119
|
-
return null;
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
function resolveTokenizerField(tokenizerConfig, ...keys) {
|
|
123
|
-
if (!tokenizerConfig) return null;
|
|
124
|
-
for (const key of keys) {
|
|
125
|
-
if (tokenizerConfig[key] != null) {
|
|
126
|
-
return tokenizerConfig[key];
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
return null;
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
function resolveConfigBoolean(rawConfig, ...keys) {
|
|
133
|
-
// Same lookup logic as resolveTokenizerField: return the first non-nullish
|
|
134
|
-
// value from the given keys. Delegates to avoid duplicating the pattern.
|
|
135
|
-
return resolveTokenizerField(rawConfig, ...keys);
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
function resolveTokenizerVocabSize(tokenizerConfig, rawConfig, architecture) {
|
|
139
|
-
const nestedTextConfig = getNestedTextConfig(rawConfig);
|
|
140
|
-
const configVocab = rawConfig?.vocab_size ?? nestedTextConfig?.vocab_size;
|
|
141
|
-
const tokenizerVocab = tokenizerConfig?.vocab_size ?? tokenizerConfig?.vocabSize;
|
|
142
|
-
const archVocab = architecture?.vocabSize;
|
|
143
|
-
|
|
144
|
-
// Warn if multiple sources provide vocab size and they disagree
|
|
145
|
-
const sources = [
|
|
146
|
-
tokenizerVocab != null ? { label: 'tokenizer', value: tokenizerVocab } : null,
|
|
147
|
-
configVocab != null ? { label: 'config', value: configVocab } : null,
|
|
148
|
-
archVocab != null ? { label: 'architecture', value: archVocab } : null,
|
|
149
|
-
].filter(Boolean);
|
|
150
|
-
if (sources.length > 1) {
|
|
151
|
-
const distinct = new Set(sources.map((s) => s.value));
|
|
152
|
-
if (distinct.size > 1) {
|
|
153
|
-
const detail = sources.map((s) => `${s.label}=${s.value}`).join(', ');
|
|
154
|
-
log.error(
|
|
155
|
-
'Convert',
|
|
156
|
-
`Vocab size sources disagree: ${detail}. Using first available (${sources[0].label}=${sources[0].value}). ` +
|
|
157
|
-
'This may cause embedding size mismatches at runtime. Verify the correct vocab size in the conversion config.'
|
|
158
|
-
);
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
return tokenizerVocab ?? configVocab ?? archVocab ?? null;
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
export function normalizeStorageQuant(value) {
|
|
166
|
-
if (value == null) return null;
|
|
167
|
-
const lower = String(value).trim().toLowerCase();
|
|
168
|
-
if (!lower) return null;
|
|
169
|
-
if (lower === 'fp16' || lower === 'float16') return 'f16';
|
|
170
|
-
if (lower === 'fp32' || lower === 'float32') return 'f32';
|
|
171
|
-
if (lower === 'bfloat16') return 'bf16';
|
|
172
|
-
if (lower === 'q4_k_m' || lower === 'q4km') return 'q4k';
|
|
173
|
-
if (lower === 'q4_0' || lower === 'q4-0') return 'q4_0';
|
|
174
|
-
if (
|
|
175
|
-
lower === 'w4a16-ct'
|
|
176
|
-
|| lower === 'w4a16_ct'
|
|
177
|
-
|| lower === 'compressed-tensors-w4a16'
|
|
178
|
-
|| lower === 'compressed_tensors_w4a16'
|
|
179
|
-
) return 'w4a16';
|
|
180
|
-
if (lower === 'wna8-o8' || lower === 'wna8_o8') return 'wna8o8';
|
|
181
|
-
return lower;
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
const SOURCE_PACKED_QUANT_DTYPES = new Set(['q4_0', 'w4a16', 'wna8o8']);
|
|
185
|
-
|
|
186
|
-
const SOURCE_PACKED_MANIFEST_DTYPES = {
|
|
187
|
-
q4_0: 'Q4_0',
|
|
188
|
-
w4a16: 'W4A16',
|
|
189
|
-
wna8o8: 'WNA8O8',
|
|
190
|
-
};
|
|
191
|
-
|
|
192
|
-
const SOURCE_PACKED_STORAGE_DESCRIPTORS = {
|
|
193
|
-
q4_0: {
|
|
194
|
-
packing: 'q4_0',
|
|
195
|
-
blockShape: [32],
|
|
196
|
-
blockBytes: 18,
|
|
197
|
-
},
|
|
198
|
-
w4a16: {
|
|
199
|
-
packing: 'w4a16',
|
|
200
|
-
blockShape: [32],
|
|
201
|
-
blockBytes: 16,
|
|
202
|
-
},
|
|
203
|
-
};
|
|
204
|
-
|
|
205
|
-
const COMPRESSED_TENSORS_W4A16_SUFFIXES = {
|
|
206
|
-
packed: '.weight_packed',
|
|
207
|
-
scale: '.weight_scale',
|
|
208
|
-
shape: '.weight_shape',
|
|
209
|
-
};
|
|
210
|
-
|
|
211
|
-
function cloneSourcePackedStorageDescriptor(targetQuant) {
|
|
212
|
-
const descriptor = SOURCE_PACKED_STORAGE_DESCRIPTORS[targetQuant];
|
|
213
|
-
if (!descriptor) return null;
|
|
214
|
-
return {
|
|
215
|
-
...descriptor,
|
|
216
|
-
blockShape: [...descriptor.blockShape],
|
|
217
|
-
...(Array.isArray(descriptor.companions)
|
|
218
|
-
? { companions: descriptor.companions.map((companion) => ({ ...companion })) }
|
|
219
|
-
: {}),
|
|
220
|
-
};
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
function resolveExplicitRoleQuant(tensor, quantizationInfo) {
|
|
224
|
-
if (!quantizationInfo || typeof quantizationInfo !== 'object') {
|
|
225
|
-
return null;
|
|
226
|
-
}
|
|
227
|
-
const role = resolveTensorRole(tensor);
|
|
228
|
-
if (role === 'embedding') {
|
|
229
|
-
return normalizeStorageQuant(quantizationInfo.embeddings ?? null);
|
|
230
|
-
}
|
|
231
|
-
if (role === 'lm_head') {
|
|
232
|
-
return normalizeStorageQuant(
|
|
233
|
-
quantizationInfo.lmHead
|
|
234
|
-
?? quantizationInfo.embeddings
|
|
235
|
-
?? null
|
|
236
|
-
);
|
|
237
|
-
}
|
|
238
|
-
if (role === 'matmul' || role === 'expert' || role === 'router') {
|
|
239
|
-
return normalizeStorageQuant(quantizationInfo.weights ?? null);
|
|
240
|
-
}
|
|
241
|
-
return null;
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
export function resolveTensorTargetQuant(tensorOrName, fallbackQuant, quantizationInfo) {
|
|
245
|
-
const fallback = normalizeStorageQuant(fallbackQuant);
|
|
246
|
-
if (!quantizationInfo || typeof quantizationInfo !== 'object') {
|
|
247
|
-
return fallback;
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
const role = resolveTensorRole(tensorOrName);
|
|
251
|
-
if (role === 'embedding') {
|
|
252
|
-
return normalizeStorageQuant(quantizationInfo.embeddings ?? fallback) ?? fallback;
|
|
253
|
-
}
|
|
254
|
-
if (role === 'lm_head') {
|
|
255
|
-
const headQuant = quantizationInfo.lmHead ?? quantizationInfo.embeddings ?? fallback;
|
|
256
|
-
return normalizeStorageQuant(headQuant) ?? fallback;
|
|
257
|
-
}
|
|
258
|
-
if (role === 'vision') {
|
|
259
|
-
return normalizeStorageQuant(quantizationInfo.vision ?? fallback) ?? fallback;
|
|
260
|
-
}
|
|
261
|
-
if (role === 'projector') {
|
|
262
|
-
return normalizeStorageQuant(quantizationInfo.projector ?? fallback) ?? fallback;
|
|
263
|
-
}
|
|
264
|
-
if (role === 'audio') {
|
|
265
|
-
return normalizeStorageQuant(quantizationInfo.audio ?? fallback) ?? fallback;
|
|
266
|
-
}
|
|
267
|
-
return normalizeStorageQuant(quantizationInfo.weights ?? fallback) ?? fallback;
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
function bf16ToFloat32(value) {
|
|
271
|
-
const view = new DataView(new ArrayBuffer(4));
|
|
272
|
-
view.setUint32(0, (value & 0xffff) << 16, true);
|
|
273
|
-
return view.getFloat32(0, true);
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
function normalizeTensorName(tensor) {
|
|
277
|
-
const name = tensor?.name;
|
|
278
|
-
return typeof name === 'string' ? name : '';
|
|
279
|
-
}
|
|
280
|
-
|
|
281
69
|
function shouldExcludeTextOnlyTensor(name) {
|
|
282
70
|
const lower = name.toLowerCase();
|
|
283
71
|
return lower.startsWith('vision_tower.')
|
|
@@ -358,202 +146,6 @@ function resolveConversionTensors(model, converterConfig) {
|
|
|
358
146
|
));
|
|
359
147
|
}
|
|
360
148
|
|
|
361
|
-
function normalizePositiveIntegerShape(value, tensorName) {
|
|
362
|
-
if (!Array.isArray(value) || value.length === 0) {
|
|
363
|
-
throw new Error(`Compressed-tensors W4A16 tensor "${tensorName}" is missing a logical shape.`);
|
|
364
|
-
}
|
|
365
|
-
return value.map((entry, index) => {
|
|
366
|
-
const number = Number(entry);
|
|
367
|
-
if (!Number.isInteger(number) || number <= 0) {
|
|
368
|
-
throw new Error(
|
|
369
|
-
`Compressed-tensors W4A16 tensor "${tensorName}" has invalid shape[${index}]=${JSON.stringify(entry)}.`
|
|
370
|
-
);
|
|
371
|
-
}
|
|
372
|
-
return number;
|
|
373
|
-
});
|
|
374
|
-
}
|
|
375
|
-
|
|
376
|
-
function resolveCompressedTensorBaseName(name, suffix) {
|
|
377
|
-
return name.endsWith(suffix)
|
|
378
|
-
? name.slice(0, -suffix.length)
|
|
379
|
-
: null;
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
function collectCompressedTensorsW4A16(tensors) {
|
|
383
|
-
const groups = new Map();
|
|
384
|
-
const groupFor = (baseName) => {
|
|
385
|
-
const existing = groups.get(baseName);
|
|
386
|
-
if (existing) return existing;
|
|
387
|
-
const created = {
|
|
388
|
-
baseName,
|
|
389
|
-
packed: null,
|
|
390
|
-
scale: null,
|
|
391
|
-
shape: null,
|
|
392
|
-
};
|
|
393
|
-
groups.set(baseName, created);
|
|
394
|
-
return created;
|
|
395
|
-
};
|
|
396
|
-
|
|
397
|
-
for (const tensor of tensors) {
|
|
398
|
-
const name = normalizeTensorName(tensor);
|
|
399
|
-
const packedBase = resolveCompressedTensorBaseName(name, COMPRESSED_TENSORS_W4A16_SUFFIXES.packed);
|
|
400
|
-
if (packedBase) {
|
|
401
|
-
groupFor(packedBase).packed = tensor;
|
|
402
|
-
continue;
|
|
403
|
-
}
|
|
404
|
-
const scaleBase = resolveCompressedTensorBaseName(name, COMPRESSED_TENSORS_W4A16_SUFFIXES.scale);
|
|
405
|
-
if (scaleBase) {
|
|
406
|
-
groupFor(scaleBase).scale = tensor;
|
|
407
|
-
continue;
|
|
408
|
-
}
|
|
409
|
-
const shapeBase = resolveCompressedTensorBaseName(name, COMPRESSED_TENSORS_W4A16_SUFFIXES.shape);
|
|
410
|
-
if (shapeBase) {
|
|
411
|
-
groupFor(shapeBase).shape = tensor;
|
|
412
|
-
}
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
return groups;
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
function assertCompressedTensorsW4A16Group(group) {
|
|
419
|
-
const missing = [];
|
|
420
|
-
if (!group.packed) missing.push('weight_packed');
|
|
421
|
-
if (!group.scale) missing.push('weight_scale');
|
|
422
|
-
if (!group.shape) missing.push('weight_shape');
|
|
423
|
-
if (missing.length > 0) {
|
|
424
|
-
throw new Error(
|
|
425
|
-
`Compressed-tensors W4A16 tensor "${group.baseName}.weight" is missing companion tensors: ${missing.join(', ')}.`
|
|
426
|
-
);
|
|
427
|
-
}
|
|
428
|
-
const packedShape = normalizePositiveIntegerShape(group.packed.shape, group.packed.name);
|
|
429
|
-
if (packedShape.length !== 2) {
|
|
430
|
-
throw new Error(
|
|
431
|
-
`Compressed-tensors W4A16 tensor "${group.packed.name}" must be 2D; got shape ${JSON.stringify(packedShape)}.`
|
|
432
|
-
);
|
|
433
|
-
}
|
|
434
|
-
const scaleShape = normalizePositiveIntegerShape(group.scale.shape, group.scale.name);
|
|
435
|
-
if (scaleShape.length !== 2) {
|
|
436
|
-
throw new Error(
|
|
437
|
-
`Compressed-tensors W4A16 scale tensor "${group.scale.name}" must be 2D; got shape ${JSON.stringify(scaleShape)}.`
|
|
438
|
-
);
|
|
439
|
-
}
|
|
440
|
-
normalizePositiveIntegerShape(group.shape.shape, group.shape.name);
|
|
441
|
-
const shapeDtype = String(group.shape.dtype || '').toUpperCase();
|
|
442
|
-
if (shapeDtype !== 'I64' && shapeDtype !== 'I32' && shapeDtype !== 'U32') {
|
|
443
|
-
throw new Error(
|
|
444
|
-
`Compressed-tensors W4A16 shape tensor "${group.shape.name}" must use I64, I32, or U32; got "${group.shape.dtype}".`
|
|
445
|
-
);
|
|
446
|
-
}
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
const COMPRESSED_TENSORS_W4A16_PACKED_VALUES_PER_ELEMENT = {
|
|
450
|
-
U8: 2,
|
|
451
|
-
I8: 2,
|
|
452
|
-
U16: 4,
|
|
453
|
-
I16: 4,
|
|
454
|
-
U32: 8,
|
|
455
|
-
I32: 8,
|
|
456
|
-
};
|
|
457
|
-
|
|
458
|
-
function inferCompressedTensorsW4A16LogicalShape(group) {
|
|
459
|
-
const packedShape = normalizePositiveIntegerShape(group.packed.shape, group.packed.name);
|
|
460
|
-
const packedDtype = String(group.packed.dtype || '').toUpperCase();
|
|
461
|
-
const valuesPerElement = COMPRESSED_TENSORS_W4A16_PACKED_VALUES_PER_ELEMENT[packedDtype];
|
|
462
|
-
if (!valuesPerElement) {
|
|
463
|
-
throw new Error(
|
|
464
|
-
`Compressed-tensors W4A16 tensor "${group.packed.name}" has unsupported packed dtype "${group.packed.dtype}".`
|
|
465
|
-
);
|
|
466
|
-
}
|
|
467
|
-
return [packedShape[0], packedShape[1] * valuesPerElement];
|
|
468
|
-
}
|
|
469
|
-
|
|
470
|
-
function shouldNormalizeCompressedTensorsW4A16(converterConfig) {
|
|
471
|
-
return normalizeStorageQuant(converterConfig?.quantization?.weights) === 'w4a16'
|
|
472
|
-
|| normalizeStorageQuant(converterConfig?.quantization?.sourceQuantizationTarget) === 'w4a16';
|
|
473
|
-
}
|
|
474
|
-
|
|
475
|
-
function sortTensorsForConversion(tensors) {
|
|
476
|
-
return [...tensors].sort((left, right) => normalizeTensorName(left).localeCompare(normalizeTensorName(right)));
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
function normalizeCompressedTensorsW4A16(tensors, converterConfig) {
|
|
480
|
-
const groups = collectCompressedTensorsW4A16(tensors);
|
|
481
|
-
if (groups.size === 0) {
|
|
482
|
-
return tensors;
|
|
483
|
-
}
|
|
484
|
-
if (!shouldNormalizeCompressedTensorsW4A16(converterConfig)) {
|
|
485
|
-
throw new Error(
|
|
486
|
-
'Compressed-tensors W4A16 tensors were detected, but converter.quantization.weights or ' +
|
|
487
|
-
'converter.quantization.sourceQuantizationTarget is not "w4a16".'
|
|
488
|
-
);
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
const byName = new Map(tensors.map((tensor) => [normalizeTensorName(tensor), tensor]));
|
|
492
|
-
const consumed = new Set();
|
|
493
|
-
const companionByName = new Map();
|
|
494
|
-
const synthetic = [];
|
|
495
|
-
const sortedGroups = [...groups.values()].sort((left, right) => left.baseName.localeCompare(right.baseName));
|
|
496
|
-
for (const group of sortedGroups) {
|
|
497
|
-
assertCompressedTensorsW4A16Group(group);
|
|
498
|
-
const logicalName = `${group.baseName}.weight`;
|
|
499
|
-
if (byName.has(logicalName)) {
|
|
500
|
-
throw new Error(
|
|
501
|
-
`Compressed-tensors W4A16 logical tensor "${logicalName}" conflicts with an existing source tensor.`
|
|
502
|
-
);
|
|
503
|
-
}
|
|
504
|
-
consumed.add(group.packed.name);
|
|
505
|
-
companionByName.set(group.scale.name, {
|
|
506
|
-
...group.scale,
|
|
507
|
-
compressedTensorsW4A16Companion: {
|
|
508
|
-
role: 'scales',
|
|
509
|
-
primary: logicalName,
|
|
510
|
-
},
|
|
511
|
-
});
|
|
512
|
-
companionByName.set(group.shape.name, {
|
|
513
|
-
...group.shape,
|
|
514
|
-
compressedTensorsW4A16Companion: {
|
|
515
|
-
role: 'shape',
|
|
516
|
-
primary: logicalName,
|
|
517
|
-
},
|
|
518
|
-
});
|
|
519
|
-
synthetic.push({
|
|
520
|
-
...group.packed,
|
|
521
|
-
name: logicalName,
|
|
522
|
-
dtype: 'W4A16',
|
|
523
|
-
shape: inferCompressedTensorsW4A16LogicalShape(group),
|
|
524
|
-
packedSourceName: group.packed.name,
|
|
525
|
-
compressedTensorsW4A16: {
|
|
526
|
-
packed: group.packed.name,
|
|
527
|
-
scales: group.scale.name,
|
|
528
|
-
shape: group.shape.name,
|
|
529
|
-
},
|
|
530
|
-
storage: {
|
|
531
|
-
packing: 'w4a16',
|
|
532
|
-
blockShape: [32],
|
|
533
|
-
blockBytes: 16,
|
|
534
|
-
companions: [
|
|
535
|
-
{ role: 'scales', tensorId: group.scale.name },
|
|
536
|
-
{ role: 'shape', tensorId: group.shape.name },
|
|
537
|
-
],
|
|
538
|
-
},
|
|
539
|
-
});
|
|
540
|
-
}
|
|
541
|
-
|
|
542
|
-
return sortTensorsForConversion([
|
|
543
|
-
...tensors
|
|
544
|
-
.filter((tensor) => !consumed.has(normalizeTensorName(tensor)))
|
|
545
|
-
.map((tensor) => companionByName.get(normalizeTensorName(tensor)) ?? tensor),
|
|
546
|
-
...synthetic,
|
|
547
|
-
]);
|
|
548
|
-
}
|
|
549
|
-
|
|
550
|
-
function isCompressedTensorsW4A16CompanionTensor(tensor) {
|
|
551
|
-
return Boolean(
|
|
552
|
-
tensor?.compressedTensorsW4A16Companion
|
|
553
|
-
&& typeof tensor.compressedTensorsW4A16Companion === 'object'
|
|
554
|
-
);
|
|
555
|
-
}
|
|
556
|
-
|
|
557
149
|
function shouldMaterializeTiedLmHead(tensors, options) {
|
|
558
150
|
if (options?.inference?.output?.tieWordEmbeddings !== true) {
|
|
559
151
|
return false;
|
|
@@ -635,1327 +227,120 @@ function materializeTiedLmHeadTensor(tensors, options) {
|
|
|
635
227
|
];
|
|
636
228
|
}
|
|
637
229
|
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
if (dtype === 'F32') {
|
|
641
|
-
if (tensorData.byteLength % 4 !== 0) {
|
|
642
|
-
throw new Error(`Invalid F32 tensor byte length for ${tensorName}: ${tensorData.byteLength}`);
|
|
643
|
-
}
|
|
644
|
-
return new Float32Array(
|
|
645
|
-
tensorData.buffer,
|
|
646
|
-
tensorData.byteOffset,
|
|
647
|
-
tensorData.byteLength / 4
|
|
648
|
-
);
|
|
649
|
-
}
|
|
650
|
-
if (dtype === 'F16') {
|
|
651
|
-
if (tensorData.byteLength % 2 !== 0) {
|
|
652
|
-
throw new Error(`Invalid F16 tensor byte length for ${tensorName}: ${tensorData.byteLength}`);
|
|
653
|
-
}
|
|
654
|
-
const f16 = new Uint16Array(
|
|
655
|
-
tensorData.buffer,
|
|
656
|
-
tensorData.byteOffset,
|
|
657
|
-
tensorData.byteLength / 2
|
|
658
|
-
);
|
|
659
|
-
const f32 = new Float32Array(f16.length);
|
|
660
|
-
for (let i = 0; i < f16.length; i++) {
|
|
661
|
-
f32[i] = float16ToFloat32(f16[i]);
|
|
662
|
-
}
|
|
663
|
-
return f32;
|
|
664
|
-
}
|
|
665
|
-
if (dtype === 'BF16') {
|
|
666
|
-
if (tensorData.byteLength % 2 !== 0) {
|
|
667
|
-
throw new Error(`Invalid BF16 tensor byte length for ${tensorName}: ${tensorData.byteLength}`);
|
|
668
|
-
}
|
|
669
|
-
const bf16 = new Uint16Array(
|
|
670
|
-
tensorData.buffer,
|
|
671
|
-
tensorData.byteOffset,
|
|
672
|
-
tensorData.byteLength / 2
|
|
673
|
-
);
|
|
674
|
-
const f32 = new Float32Array(bf16.length);
|
|
675
|
-
for (let i = 0; i < bf16.length; i++) {
|
|
676
|
-
f32[i] = bf16ToFloat32(bf16[i]);
|
|
677
|
-
}
|
|
678
|
-
return f32;
|
|
679
|
-
}
|
|
680
|
-
throw new Error(`Cannot quantize ${tensorName} from ${dtype} to Q4_K_M`);
|
|
681
|
-
}
|
|
682
|
-
|
|
683
|
-
function resolveConfigTokenId(rawConfig, key) {
|
|
684
|
-
const direct = rawConfig?.[key];
|
|
685
|
-
const nested = getNestedTextConfig(rawConfig)?.[key];
|
|
686
|
-
return resolveTokenizerId(direct ?? nested);
|
|
687
|
-
}
|
|
688
|
-
|
|
689
|
-
function resolveConfigTokenIds(rawConfig, key) {
|
|
690
|
-
const direct = rawConfig?.[key];
|
|
691
|
-
const nested = getNestedTextConfig(rawConfig)?.[key];
|
|
692
|
-
return resolveTokenizerIds(direct ?? nested);
|
|
693
|
-
}
|
|
230
|
+
// Re-export formatBytes from schema for backward compatibility
|
|
231
|
+
export { formatBytes };
|
|
694
232
|
|
|
695
|
-
function
|
|
696
|
-
const
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
if (Number.isFinite(direct) && direct > 0) return Number(direct);
|
|
700
|
-
const nested = nestedTextConfig?.[key];
|
|
701
|
-
if (Number.isFinite(nested) && nested > 0) return Number(nested);
|
|
233
|
+
function validateInt4PleMaterializationContract(tensorLocations, inference, modelId) {
|
|
234
|
+
const materialization = inference?.session?.perLayerInputs?.materialization;
|
|
235
|
+
if (materialization !== 'gpu_split_tables') {
|
|
236
|
+
return;
|
|
702
237
|
}
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
function normalizeTensorShape(value) {
|
|
707
|
-
if (!Array.isArray(value) || value.length !== 2) return null;
|
|
708
|
-
const rows = Number(value[0]);
|
|
709
|
-
const cols = Number(value[1]);
|
|
710
|
-
if (!Number.isFinite(rows) || !Number.isFinite(cols)) return null;
|
|
711
|
-
if (rows <= 0 || cols <= 0) return null;
|
|
712
|
-
return [Math.trunc(rows), Math.trunc(cols)];
|
|
713
|
-
}
|
|
714
|
-
|
|
715
|
-
function isExpertTensorName(name) {
|
|
716
|
-
const lower = String(name || '').toLowerCase();
|
|
717
|
-
return lower.includes('.experts.') || lower.includes('.expert.') || lower.includes('block_sparse_moe');
|
|
718
|
-
}
|
|
719
|
-
|
|
720
|
-
function inferDenseIntermediateSizeFromTensorEntries(entries) {
|
|
721
|
-
if (!Array.isArray(entries) || entries.length === 0) return null;
|
|
722
|
-
const candidates = [];
|
|
723
|
-
for (const entry of entries) {
|
|
724
|
-
const name = String(entry?.name || '');
|
|
725
|
-
if (!name || isExpertTensorName(name)) continue;
|
|
726
|
-
const shape = normalizeTensorShape(entry?.shape);
|
|
727
|
-
if (!shape) continue;
|
|
728
|
-
const lower = name.toLowerCase();
|
|
729
|
-
if (
|
|
730
|
-
lower.endsWith('.feed_forward.w1.weight')
|
|
731
|
-
|| lower.endsWith('.feed_forward.w3.weight')
|
|
732
|
-
|| lower.endsWith('.ffn_gate.weight')
|
|
733
|
-
|| lower.endsWith('.ffn_up.weight')
|
|
734
|
-
|| lower.endsWith('.ffn.gate_proj.weight')
|
|
735
|
-
|| lower.endsWith('.ffn.up_proj.weight')
|
|
736
|
-
|| lower.endsWith('.mlp.gate_proj.weight')
|
|
737
|
-
|| lower.endsWith('.mlp.up_proj.weight')
|
|
738
|
-
) {
|
|
739
|
-
candidates.push(shape[0]);
|
|
740
|
-
continue;
|
|
741
|
-
}
|
|
742
|
-
if (
|
|
743
|
-
lower.endsWith('.feed_forward.w2.weight')
|
|
744
|
-
|| lower.endsWith('.ffn_down.weight')
|
|
745
|
-
|| lower.endsWith('.ffn.down_proj.weight')
|
|
746
|
-
|| lower.endsWith('.mlp.down_proj.weight')
|
|
747
|
-
) {
|
|
748
|
-
candidates.push(shape[1]);
|
|
749
|
-
continue;
|
|
750
|
-
}
|
|
238
|
+
for (const [name, location] of Object.entries(tensorLocations ?? {})) {
|
|
239
|
+
const sourceTransform = location?.sourceTransform ?? null;
|
|
751
240
|
if (
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|| lower.endsWith('.mlp.gate_up_proj.weight')
|
|
241
|
+
isGemma4PerLayerEmbedTensor(name)
|
|
242
|
+
&& sourceTransform?.kind === 'litert_axis_dequant'
|
|
243
|
+
&& String(sourceTransform?.sourceDtype ?? '').toUpperCase() === 'INT4'
|
|
756
244
|
) {
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
245
|
+
throw new Error(
|
|
246
|
+
`Manifest "${modelId}" Gemma 4 INT4 PLE tensor "${name}" cannot use ` +
|
|
247
|
+
'inference.session.perLayerInputs.materialization="gpu_split_tables". ' +
|
|
248
|
+
'Use materialization="range_backed" or disable INT4 PLE quantization.'
|
|
249
|
+
);
|
|
760
250
|
}
|
|
761
251
|
}
|
|
762
|
-
if (candidates.length === 0) return null;
|
|
763
|
-
const counts = new Map();
|
|
764
|
-
for (const value of candidates) {
|
|
765
|
-
counts.set(value, (counts.get(value) ?? 0) + 1);
|
|
766
|
-
}
|
|
767
|
-
return [...counts.entries()]
|
|
768
|
-
.sort((a, b) => {
|
|
769
|
-
if (b[1] !== a[1]) return b[1] - a[1];
|
|
770
|
-
return a[0] - b[0];
|
|
771
|
-
})[0]?.[0] ?? null;
|
|
772
252
|
}
|
|
773
253
|
|
|
774
|
-
function
|
|
775
|
-
|
|
776
|
-
const
|
|
777
|
-
if (
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
const entries = Array.isArray(model?.tensors) && model.tensors.length > 0
|
|
785
|
-
? model.tensors
|
|
786
|
-
: Object.entries(tensorLocations ?? {}).map(([name, location]) => ({ name, shape: location?.shape }));
|
|
787
|
-
const inferred = inferDenseIntermediateSizeFromTensorEntries(entries);
|
|
788
|
-
if (inferred == null || inferred === current) {
|
|
789
|
-
return architecture;
|
|
254
|
+
function resolveGemma4TextConfig(rawConfig) {
|
|
255
|
+
const textConfig = getNestedTextConfig(rawConfig);
|
|
256
|
+
const modelType = String(textConfig?.model_type ?? rawConfig?.model_type ?? '').trim().toLowerCase();
|
|
257
|
+
if (
|
|
258
|
+
modelType !== 'gemma4'
|
|
259
|
+
&& modelType !== 'gemma4_text'
|
|
260
|
+
&& modelType !== 'gemma4_unified'
|
|
261
|
+
&& modelType !== 'gemma4_unified_text'
|
|
262
|
+
) {
|
|
263
|
+
return null;
|
|
790
264
|
}
|
|
791
|
-
|
|
792
|
-
'Convert',
|
|
793
|
-
`Adjusted architecture.intermediateSize for "${modelId}": ${current} -> ${inferred} (from FFN tensor shapes)`
|
|
794
|
-
);
|
|
795
|
-
return {
|
|
796
|
-
...architecture,
|
|
797
|
-
intermediateSize: inferred,
|
|
798
|
-
};
|
|
799
|
-
}
|
|
800
|
-
|
|
801
|
-
function modelHasMoETensors(model) {
|
|
802
|
-
if (!Array.isArray(model?.tensors)) return false;
|
|
803
|
-
return model.tensors.some((tensor) => {
|
|
804
|
-
const name = String(tensor?.name || '').toLowerCase();
|
|
805
|
-
return (
|
|
806
|
-
name.includes('.experts.') ||
|
|
807
|
-
name.includes('.expert.') ||
|
|
808
|
-
name.includes('block_sparse_moe')
|
|
809
|
-
);
|
|
810
|
-
});
|
|
265
|
+
return textConfig ?? rawConfig ?? null;
|
|
811
266
|
}
|
|
812
267
|
|
|
813
|
-
function
|
|
814
|
-
|
|
815
|
-
const
|
|
816
|
-
if (
|
|
817
|
-
|
|
818
|
-
}
|
|
819
|
-
const modelType = String(
|
|
820
|
-
resolvedModelType ??
|
|
821
|
-
rawConfig?.model_type ??
|
|
822
|
-
getNestedTextConfig(rawConfig)?.model_type ??
|
|
823
|
-
''
|
|
824
|
-
).toLowerCase();
|
|
825
|
-
if (modelType.includes('gpt_oss') || modelType.includes('gpt-oss') || modelType.includes('gptoss')) {
|
|
826
|
-
return 'gpt-oss';
|
|
827
|
-
}
|
|
828
|
-
if (modelType === 'diffusion_gemma' || modelType === 'diffusion_gemma_text') {
|
|
829
|
-
return 'gemma4';
|
|
268
|
+
function collectGemma4UnsupportedTensorFlags(tensors) {
|
|
269
|
+
const names = Array.isArray(tensors) ? tensors.map((tensor) => normalizeTensorName(tensor).toLowerCase()) : [];
|
|
270
|
+
const flags = [];
|
|
271
|
+
if (names.some((name) => name.includes('.experts.gate_up_proj'))) {
|
|
272
|
+
flags.push('experts.gate_up_proj');
|
|
830
273
|
}
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
function normalizeMoEConfig(config, contextLabel) {
|
|
835
|
-
if (!config) return null;
|
|
836
|
-
const numExperts = Number(config.numExperts);
|
|
837
|
-
const numExpertsPerToken = Number(config.numExpertsPerToken);
|
|
838
|
-
const expertFormat = String(config.expertFormat || '').trim();
|
|
839
|
-
const expertIntermediateSize = config.expertIntermediateSize == null
|
|
840
|
-
? null
|
|
841
|
-
: Number(config.expertIntermediateSize);
|
|
842
|
-
const allowedExpertFormat = expertFormat === 'gpt-oss' || expertFormat === 'mixtral' || expertFormat === 'gemma4';
|
|
843
|
-
if (!Number.isFinite(numExperts) || numExperts <= 0) {
|
|
844
|
-
throw new Error(`Invalid moeConfig.numExperts for ${contextLabel}`);
|
|
274
|
+
if (names.some((name) => name.includes('.experts.down_proj'))) {
|
|
275
|
+
flags.push('experts.down_proj');
|
|
845
276
|
}
|
|
846
|
-
if (
|
|
847
|
-
|
|
277
|
+
if (names.some((name) => name.includes('.router.per_expert_scale'))) {
|
|
278
|
+
flags.push('router.per_expert_scale');
|
|
848
279
|
}
|
|
849
|
-
if (
|
|
850
|
-
|
|
280
|
+
if (names.some((name) => name.includes('.router.scale'))) {
|
|
281
|
+
flags.push('router.scale');
|
|
851
282
|
}
|
|
852
|
-
if (
|
|
853
|
-
|
|
283
|
+
if (names.some((name) => name.includes('.post_feedforward_layernorm_1.'))) {
|
|
284
|
+
flags.push('post_feedforward_layernorm_1');
|
|
854
285
|
}
|
|
855
|
-
if (
|
|
856
|
-
|
|
857
|
-
&& (!Number.isFinite(expertIntermediateSize) || expertIntermediateSize <= 0)
|
|
858
|
-
) {
|
|
859
|
-
throw new Error(`Invalid moeConfig.expertIntermediateSize for ${contextLabel}`);
|
|
286
|
+
if (names.some((name) => name.includes('.post_feedforward_layernorm_2.'))) {
|
|
287
|
+
flags.push('post_feedforward_layernorm_2');
|
|
860
288
|
}
|
|
861
|
-
if (
|
|
862
|
-
|
|
289
|
+
if (names.some((name) => name.includes('.pre_feedforward_layernorm_2.'))) {
|
|
290
|
+
flags.push('pre_feedforward_layernorm_2');
|
|
863
291
|
}
|
|
864
|
-
return
|
|
865
|
-
numExperts,
|
|
866
|
-
numExpertsPerToken,
|
|
867
|
-
expertFormat,
|
|
868
|
-
...(expertIntermediateSize == null ? {} : { expertIntermediateSize }),
|
|
869
|
-
};
|
|
292
|
+
return flags;
|
|
870
293
|
}
|
|
871
294
|
|
|
872
|
-
|
|
873
|
-
const
|
|
874
|
-
|
|
295
|
+
function assertSupportedGemma4Conversion(model, tensors, modelId) {
|
|
296
|
+
const rawConfig = model?.config ?? null;
|
|
297
|
+
const textConfig = resolveGemma4TextConfig(rawConfig);
|
|
298
|
+
if (!textConfig) return;
|
|
875
299
|
|
|
876
|
-
const
|
|
877
|
-
|
|
300
|
+
const hiddenSizePerLayerInput = Number(textConfig.hidden_size_per_layer_input ?? 0);
|
|
301
|
+
if (Number.isFinite(hiddenSizePerLayerInput) && hiddenSizePerLayerInput > 0) {
|
|
302
|
+
const names = Array.isArray(tensors)
|
|
303
|
+
? tensors.map((tensor) => String(tensor?.name ?? '').trim())
|
|
304
|
+
: [];
|
|
305
|
+
const requiredNames = [
|
|
306
|
+
'embed_tokens_per_layer.weight',
|
|
307
|
+
'per_layer_input_gate.weight',
|
|
308
|
+
'per_layer_projection.weight',
|
|
309
|
+
'post_per_layer_input_norm.weight',
|
|
310
|
+
'per_layer_model_projection.weight',
|
|
311
|
+
'per_layer_projection_norm.weight',
|
|
312
|
+
];
|
|
313
|
+
const missing = requiredNames.filter((suffix) => !names.some((name) => name.endsWith(suffix)));
|
|
314
|
+
if (missing.length > 0) {
|
|
315
|
+
throw new Error(
|
|
316
|
+
`Gemma 4 model "${modelId}" declares hidden_size_per_layer_input=${hiddenSizePerLayerInput}, ` +
|
|
317
|
+
`but the checkpoint is missing required per-layer input tensors: ${missing.join(', ')}.`
|
|
318
|
+
);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
878
321
|
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
if (!hasMoETensors && (!numExperts || numExperts <= 1)) {
|
|
882
|
-
return null;
|
|
322
|
+
if (textConfig.enable_moe_block !== true) {
|
|
323
|
+
return;
|
|
883
324
|
}
|
|
884
325
|
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
);
|
|
326
|
+
const unsupportedFlags = collectGemma4UnsupportedTensorFlags(tensors);
|
|
327
|
+
if (unsupportedFlags.length === 0 && !modelHasMoETensors({ tensors })) {
|
|
328
|
+
return;
|
|
889
329
|
}
|
|
890
330
|
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
'
|
|
894
|
-
'
|
|
895
|
-
'
|
|
896
|
-
'experts_per_token',
|
|
897
|
-
'expertUsedCount'
|
|
331
|
+
throw new Error(
|
|
332
|
+
`Gemma 4 model "${modelId}" is not supported yet: Gemma 4 MoE decoder blocks require ` +
|
|
333
|
+
'Gemma-specific router scaling and dual dense+MoE FFN execution, but current Doppler MoE runtime ' +
|
|
334
|
+
'only supports Mixtral/GPT-OSS semantics. ' +
|
|
335
|
+
`Detected: ${unsupportedFlags.length > 0 ? unsupportedFlags.join(', ') : 'Gemma 4 MoE tensors'}.`
|
|
898
336
|
);
|
|
337
|
+
}
|
|
899
338
|
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
'(expected top_k_experts/num_experts_per_tok/num_experts_per_token/experts_per_token)'
|
|
904
|
-
);
|
|
339
|
+
export function buildTensorMap(tensors, shardSize) {
|
|
340
|
+
if (!shardSize || shardSize <= 0) {
|
|
341
|
+
throw new Error('Missing shard size for tensor map');
|
|
905
342
|
}
|
|
906
|
-
|
|
907
|
-
const expertFormat = resolveMoEExpertFormat(
|
|
908
|
-
rawConfig,
|
|
909
|
-
resolvedModelType,
|
|
910
|
-
options?.quantizationInfo ?? null,
|
|
911
|
-
null
|
|
912
|
-
);
|
|
913
|
-
const expertIntermediateSize = expertFormat === 'gemma4'
|
|
914
|
-
? resolveMoEConfigNumber(rawConfig, 'moe_intermediate_size', 'expert_intermediate_size')
|
|
915
|
-
: null;
|
|
916
|
-
|
|
917
|
-
return normalizeMoEConfig(
|
|
918
|
-
{
|
|
919
|
-
numExperts,
|
|
920
|
-
numExpertsPerToken,
|
|
921
|
-
expertFormat,
|
|
922
|
-
...(expertIntermediateSize == null ? {} : { expertIntermediateSize }),
|
|
923
|
-
},
|
|
924
|
-
options?.modelId ?? 'model'
|
|
925
|
-
);
|
|
926
|
-
}
|
|
927
|
-
|
|
928
|
-
export function buildSentencepieceTokenizer(tokenizerConfig, rawConfig, architecture, modelTokenizerModel) {
|
|
929
|
-
if (!modelTokenizerModel) return null;
|
|
930
|
-
|
|
931
|
-
const vocabSize = resolveTokenizerVocabSize(tokenizerConfig, rawConfig, architecture);
|
|
932
|
-
const sentencepieceModel = typeof modelTokenizerModel === 'string'
|
|
933
|
-
? modelTokenizerModel
|
|
934
|
-
: modelTokenizerModel?.file ?? 'tokenizer.model';
|
|
935
|
-
|
|
936
|
-
const bosTokenId = resolveTokenizerId(
|
|
937
|
-
resolveTokenizerField(tokenizerConfig, 'bos_token_id', 'bosTokenId')
|
|
938
|
-
?? resolveConfigTokenId(rawConfig, 'bos_token_id')
|
|
939
|
-
);
|
|
940
|
-
const eosTokenId = resolveTokenizerId(
|
|
941
|
-
resolveTokenizerField(tokenizerConfig, 'eos_token_id', 'eosTokenId')
|
|
942
|
-
?? resolveConfigTokenId(rawConfig, 'eos_token_id')
|
|
943
|
-
);
|
|
944
|
-
const eosTokens = resolveTokenizerIds(
|
|
945
|
-
resolveTokenizerField(tokenizerConfig, 'eos_token_ids', 'eosTokens', 'eos_token_id')
|
|
946
|
-
?? resolveConfigTokenIds(rawConfig, 'eos_token_ids')
|
|
947
|
-
);
|
|
948
|
-
const padTokenId = resolveTokenizerId(
|
|
949
|
-
resolveTokenizerField(tokenizerConfig, 'pad_token_id', 'padTokenId')
|
|
950
|
-
?? resolveConfigTokenId(rawConfig, 'pad_token_id')
|
|
951
|
-
);
|
|
952
|
-
const unkTokenId = resolveTokenizerId(
|
|
953
|
-
resolveTokenizerField(tokenizerConfig, 'unk_token_id', 'unkTokenId')
|
|
954
|
-
?? resolveConfigTokenId(rawConfig, 'unk_token_id')
|
|
955
|
-
);
|
|
956
|
-
const addBosToken = resolveTokenizerField(tokenizerConfig, 'add_bos_token', 'addBosToken');
|
|
957
|
-
const addEosToken = resolveTokenizerField(tokenizerConfig, 'add_eos_token', 'addEosToken');
|
|
958
|
-
|
|
959
|
-
const tokenizer = {
|
|
960
|
-
type: 'sentencepiece',
|
|
961
|
-
sentencepieceModel,
|
|
962
|
-
vocabSize: vocabSize ?? 0,
|
|
963
|
-
};
|
|
964
|
-
|
|
965
|
-
if (bosTokenId != null) tokenizer.bosTokenId = bosTokenId;
|
|
966
|
-
if (eosTokenId != null) tokenizer.eosTokenId = eosTokenId;
|
|
967
|
-
if (eosTokens) tokenizer.eosTokens = eosTokens;
|
|
968
|
-
if (padTokenId != null) tokenizer.padTokenId = padTokenId;
|
|
969
|
-
if (unkTokenId != null) tokenizer.unkTokenId = unkTokenId;
|
|
970
|
-
if (addBosToken != null) tokenizer.addBosToken = addBosToken;
|
|
971
|
-
if (addEosToken != null) tokenizer.addEosToken = addEosToken;
|
|
972
|
-
|
|
973
|
-
return tokenizer;
|
|
974
|
-
}
|
|
975
|
-
|
|
976
|
-
export function resolveBundledTokenizerVocabSize(tokenizerJson) {
|
|
977
|
-
const vocab = tokenizerJson?.model?.vocab;
|
|
978
|
-
if (Array.isArray(vocab)) {
|
|
979
|
-
return vocab.length;
|
|
980
|
-
}
|
|
981
|
-
if (vocab && typeof vocab === 'object') {
|
|
982
|
-
return Object.keys(vocab).length;
|
|
983
|
-
}
|
|
984
|
-
return 0;
|
|
985
|
-
}
|
|
986
|
-
|
|
987
|
-
export function buildBundledTokenizer(tokenizerJson, tokenizerConfig, rawConfig) {
|
|
988
|
-
const vocabSize = resolveBundledTokenizerVocabSize(tokenizerJson);
|
|
989
|
-
if (!vocabSize) {
|
|
990
|
-
throw new Error('Tokenizer vocab is missing or empty');
|
|
991
|
-
}
|
|
992
|
-
|
|
993
|
-
const tokenizer = {
|
|
994
|
-
type: 'bundled',
|
|
995
|
-
vocabSize,
|
|
996
|
-
file: 'tokenizer.json',
|
|
997
|
-
};
|
|
998
|
-
const addBosToken = (
|
|
999
|
-
resolveTokenizerField(tokenizerJson, 'add_bos_token', 'addBosToken')
|
|
1000
|
-
?? resolveTokenizerField(tokenizerConfig, 'add_bos_token', 'addBosToken')
|
|
1001
|
-
?? resolveConfigBoolean(rawConfig, 'add_bos_token', 'addBosToken')
|
|
1002
|
-
);
|
|
1003
|
-
const addEosToken = (
|
|
1004
|
-
resolveTokenizerField(tokenizerJson, 'add_eos_token', 'addEosToken')
|
|
1005
|
-
?? resolveTokenizerField(tokenizerConfig, 'add_eos_token', 'addEosToken')
|
|
1006
|
-
?? resolveConfigBoolean(rawConfig, 'add_eos_token', 'addEosToken')
|
|
1007
|
-
);
|
|
1008
|
-
const inferredFlags = inferBundledTokenizerBehaviorFlags(tokenizerJson);
|
|
1009
|
-
|
|
1010
|
-
if (addBosToken != null) tokenizer.addBosToken = addBosToken;
|
|
1011
|
-
else if (inferredFlags.addBosToken != null) tokenizer.addBosToken = inferredFlags.addBosToken;
|
|
1012
|
-
if (addEosToken != null) tokenizer.addEosToken = addEosToken;
|
|
1013
|
-
else if (inferredFlags.addEosToken != null) tokenizer.addEosToken = inferredFlags.addEosToken;
|
|
1014
|
-
|
|
1015
|
-
return tokenizer;
|
|
1016
|
-
}
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
export function sanitizeModelId(name) {
|
|
1020
|
-
const sanitized = name
|
|
1021
|
-
.toLowerCase()
|
|
1022
|
-
.replace(/[^a-z0-9_-]/g, '-')
|
|
1023
|
-
.replace(/-+/g, '-')
|
|
1024
|
-
.replace(/^-|-$/g, '')
|
|
1025
|
-
.slice(0, 64);
|
|
1026
|
-
return sanitized || null;
|
|
1027
|
-
}
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
// Re-export formatBytes from schema for backward compatibility
|
|
1031
|
-
export { formatBytes };
|
|
1032
|
-
|
|
1033
|
-
const BF16_ROUND_VIEW = new DataView(new ArrayBuffer(4));
|
|
1034
|
-
|
|
1035
|
-
function float32ToBFloat16(value) {
|
|
1036
|
-
BF16_ROUND_VIEW.setFloat32(0, value, true);
|
|
1037
|
-
const bits = BF16_ROUND_VIEW.getUint32(0, true);
|
|
1038
|
-
const lsb = (bits >> 16) & 1;
|
|
1039
|
-
const roundingBias = 0x7fff + lsb;
|
|
1040
|
-
return ((bits + roundingBias) >> 16) & 0xffff;
|
|
1041
|
-
}
|
|
1042
|
-
|
|
1043
|
-
function resolveQuantizeEmbeddings(quantizationInfo, explicitValue = null) {
|
|
1044
|
-
if (typeof explicitValue === 'boolean') {
|
|
1045
|
-
return explicitValue;
|
|
1046
|
-
}
|
|
1047
|
-
return (
|
|
1048
|
-
normalizeStorageQuant(quantizationInfo?.embeddings ?? null) === 'q4k'
|
|
1049
|
-
|| normalizeStorageQuant(quantizationInfo?.lmHead ?? null) === 'q4k'
|
|
1050
|
-
);
|
|
1051
|
-
}
|
|
1052
|
-
|
|
1053
|
-
function normalizeModulesToNotConvert(modulesToNotConvert) {
|
|
1054
|
-
if (!Array.isArray(modulesToNotConvert)) {
|
|
1055
|
-
return null;
|
|
1056
|
-
}
|
|
1057
|
-
const normalized = modulesToNotConvert
|
|
1058
|
-
.map((value) => (
|
|
1059
|
-
typeof value === 'string' ? value.trim() : ''
|
|
1060
|
-
))
|
|
1061
|
-
.filter(Boolean);
|
|
1062
|
-
return normalized.length > 0 ? normalized : null;
|
|
1063
|
-
}
|
|
1064
|
-
|
|
1065
|
-
function shouldSkipModuleQuantization(tensorName, modulesToNotConvert) {
|
|
1066
|
-
const patterns = normalizeModulesToNotConvert(modulesToNotConvert);
|
|
1067
|
-
if (!patterns) {
|
|
1068
|
-
return false;
|
|
1069
|
-
}
|
|
1070
|
-
|
|
1071
|
-
for (const pattern of patterns) {
|
|
1072
|
-
const regexPattern = pattern
|
|
1073
|
-
.replace(/\./g, '\\.')
|
|
1074
|
-
.replace(/\*/g, '\\d+');
|
|
1075
|
-
const matcher = new RegExp(regexPattern);
|
|
1076
|
-
if (matcher.test(tensorName)) {
|
|
1077
|
-
return true;
|
|
1078
|
-
}
|
|
1079
|
-
}
|
|
1080
|
-
return false;
|
|
1081
|
-
}
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
export function shouldQuantize(tensorName, shape, options = {}) {
|
|
1085
|
-
const {
|
|
1086
|
-
quantizeEmbeddings = false,
|
|
1087
|
-
modulesToNotConvert = null,
|
|
1088
|
-
role: explicitRole = null,
|
|
1089
|
-
} = options;
|
|
1090
|
-
|
|
1091
|
-
if (!shape || !Array.isArray(shape) || shape.length === 0) {
|
|
1092
|
-
log.warn('Convert', `Invalid shape for tensor "${tensorName}": ${JSON.stringify(shape)}`);
|
|
1093
|
-
return false;
|
|
1094
|
-
}
|
|
1095
|
-
const numElements = shape.reduce((a, b) => a * b, 1);
|
|
1096
|
-
const role = typeof explicitRole === 'string' && explicitRole.trim()
|
|
1097
|
-
? explicitRole.trim()
|
|
1098
|
-
: classifyTensorRole(tensorName);
|
|
1099
|
-
const lower = tensorName.toLowerCase();
|
|
1100
|
-
const isBias = lower.endsWith('.bias') || lower.endsWith('_bias');
|
|
1101
|
-
|
|
1102
|
-
const shouldQuantizeByRole = selectRuleValue('converter', 'tensorRoles', 'shouldQuantize', {
|
|
1103
|
-
numElements,
|
|
1104
|
-
role,
|
|
1105
|
-
isBias,
|
|
1106
|
-
quantizeEmbeddings,
|
|
1107
|
-
});
|
|
1108
|
-
|
|
1109
|
-
if (!shouldQuantizeByRole) {
|
|
1110
|
-
return false;
|
|
1111
|
-
}
|
|
1112
|
-
|
|
1113
|
-
if (shouldSkipModuleQuantization(tensorName, modulesToNotConvert)) {
|
|
1114
|
-
return false;
|
|
1115
|
-
}
|
|
1116
|
-
|
|
1117
|
-
return true;
|
|
1118
|
-
}
|
|
1119
|
-
|
|
1120
|
-
const GEMMA4_PLE_TENSOR_SUFFIXES = [
|
|
1121
|
-
'embed_tokens_per_layer.weight',
|
|
1122
|
-
'per_layer_embeddings.weight',
|
|
1123
|
-
// GGUF (unsloth / ggml-org) naming for Gemma 4's PLE table.
|
|
1124
|
-
'per_layer_token_embd.weight',
|
|
1125
|
-
];
|
|
1126
|
-
|
|
1127
|
-
export function isGemma4PerLayerEmbedTensor(tensorName) {
|
|
1128
|
-
if (typeof tensorName !== 'string') return false;
|
|
1129
|
-
const normalized = tensorName.trim().toLowerCase();
|
|
1130
|
-
return GEMMA4_PLE_TENSOR_SUFFIXES.some((suffix) => normalized.endsWith(suffix));
|
|
1131
|
-
}
|
|
1132
|
-
|
|
1133
|
-
function validateInt4PleMaterializationContract(tensorLocations, inference, modelId) {
|
|
1134
|
-
const materialization = inference?.session?.perLayerInputs?.materialization;
|
|
1135
|
-
if (materialization !== 'gpu_split_tables') {
|
|
1136
|
-
return;
|
|
1137
|
-
}
|
|
1138
|
-
for (const [name, location] of Object.entries(tensorLocations ?? {})) {
|
|
1139
|
-
const sourceTransform = location?.sourceTransform ?? null;
|
|
1140
|
-
if (
|
|
1141
|
-
isGemma4PerLayerEmbedTensor(name)
|
|
1142
|
-
&& sourceTransform?.kind === 'litert_axis_dequant'
|
|
1143
|
-
&& String(sourceTransform?.sourceDtype ?? '').toUpperCase() === 'INT4'
|
|
1144
|
-
) {
|
|
1145
|
-
throw new Error(
|
|
1146
|
-
`Manifest "${modelId}" Gemma 4 INT4 PLE tensor "${name}" cannot use ` +
|
|
1147
|
-
'inference.session.perLayerInputs.materialization="gpu_split_tables". ' +
|
|
1148
|
-
'Use materialization="range_backed" or disable INT4 PLE quantization.'
|
|
1149
|
-
);
|
|
1150
|
-
}
|
|
1151
|
-
}
|
|
1152
|
-
}
|
|
1153
|
-
|
|
1154
|
-
function resolveOriginalTensorShape(options) {
|
|
1155
|
-
const shape = options?.originalTensorShape;
|
|
1156
|
-
if (!Array.isArray(shape) || shape.length !== 2) {
|
|
1157
|
-
return null;
|
|
1158
|
-
}
|
|
1159
|
-
const rows = Number(shape[0]);
|
|
1160
|
-
const cols = Number(shape[1]);
|
|
1161
|
-
if (!Number.isInteger(rows) || rows <= 0 || !Number.isInteger(cols) || cols <= 0) {
|
|
1162
|
-
return null;
|
|
1163
|
-
}
|
|
1164
|
-
return [rows, cols];
|
|
1165
|
-
}
|
|
1166
|
-
|
|
1167
|
-
function resolvePerLayerEmbeddingQuant(options) {
|
|
1168
|
-
const value = (
|
|
1169
|
-
options?.perLayerEmbeddings
|
|
1170
|
-
?? options?.quantizationInfo?.perLayerEmbeddings
|
|
1171
|
-
?? null
|
|
1172
|
-
);
|
|
1173
|
-
if (value == null) return null;
|
|
1174
|
-
return String(value).trim().toLowerCase().replace(/\s+/g, '_') || null;
|
|
1175
|
-
}
|
|
1176
|
-
|
|
1177
|
-
function canInt4QuantizePerRow(tensor, options) {
|
|
1178
|
-
if (options?.skipInt4PlePerRow === true) return false;
|
|
1179
|
-
if (resolvePerLayerEmbeddingQuant(options) !== 'int4_per_row') return false;
|
|
1180
|
-
if (!Array.isArray(tensor.shape) || tensor.shape.length !== 2) return false;
|
|
1181
|
-
const [rows, cols] = tensor.shape;
|
|
1182
|
-
if (!Number.isInteger(rows) || rows <= 0) return false;
|
|
1183
|
-
if (!Number.isInteger(cols) || cols <= 0) return false;
|
|
1184
|
-
if ((cols & 1) !== 0) return false;
|
|
1185
|
-
const originalShape = resolveOriginalTensorShape(options);
|
|
1186
|
-
if (originalShape && originalShape[1] !== cols) return false;
|
|
1187
|
-
// Per-row symmetric INT4 produces one scale per row. PLE semantics require
|
|
1188
|
-
// one scale per VOCAB token (typically 262144 for Gemma 4). HF stores PLE
|
|
1189
|
-
// as [vocab, hidden] (rows = vocab ≫ cols). GGUF stores it transposed as
|
|
1190
|
-
// [hidden, vocab] (rows = hidden ≪ cols). Quantizing the GGUF layout with
|
|
1191
|
-
// per-row would wrongly share a single scale across all vocab tokens.
|
|
1192
|
-
// Require the tensor to already be in [vocab, hidden] layout. Callers that
|
|
1193
|
-
// have a GGUF PLE must transpose before reaching this function.
|
|
1194
|
-
if (rows <= cols && (!originalShape || originalShape[0] <= originalShape[1])) return false;
|
|
1195
|
-
const srcDtype = String(tensor.dtype || '').toUpperCase();
|
|
1196
|
-
return srcDtype === 'F32' || srcDtype === 'F16' || srcDtype === 'BF16';
|
|
1197
|
-
}
|
|
1198
|
-
|
|
1199
|
-
function toFloat32FromTensor(bytes, sourceDtype, tensorName) {
|
|
1200
|
-
const src = String(sourceDtype || '').toUpperCase();
|
|
1201
|
-
if (src === 'F32') {
|
|
1202
|
-
if (bytes.byteLength % 4 !== 0) {
|
|
1203
|
-
throw new Error(`Invalid F32 tensor byte length for ${tensorName}: ${bytes.byteLength}`);
|
|
1204
|
-
}
|
|
1205
|
-
return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
|
|
1206
|
-
}
|
|
1207
|
-
if (src === 'F16') {
|
|
1208
|
-
if (bytes.byteLength % 2 !== 0) {
|
|
1209
|
-
throw new Error(`Invalid F16 tensor byte length for ${tensorName}: ${bytes.byteLength}`);
|
|
1210
|
-
}
|
|
1211
|
-
const src16 = new Uint16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2);
|
|
1212
|
-
const out = new Float32Array(src16.length);
|
|
1213
|
-
for (let i = 0; i < src16.length; i++) out[i] = float16ToFloat32(src16[i]);
|
|
1214
|
-
return out;
|
|
1215
|
-
}
|
|
1216
|
-
if (src === 'BF16') {
|
|
1217
|
-
if (bytes.byteLength % 2 !== 0) {
|
|
1218
|
-
throw new Error(`Invalid BF16 tensor byte length for ${tensorName}: ${bytes.byteLength}`);
|
|
1219
|
-
}
|
|
1220
|
-
const src16 = new Uint16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2);
|
|
1221
|
-
const out = new Float32Array(src16.length);
|
|
1222
|
-
for (let i = 0; i < src16.length; i++) out[i] = bf16ToFloat32(src16[i]);
|
|
1223
|
-
return out;
|
|
1224
|
-
}
|
|
1225
|
-
throw new Error(`Unsupported source dtype "${sourceDtype}" for PLE INT4 quantization of ${tensorName}`);
|
|
1226
|
-
}
|
|
1227
|
-
|
|
1228
|
-
function buildInt4PerRowPleTransform(tensor, bytes, sourceDtype, options = {}) {
|
|
1229
|
-
const f32 = toFloat32FromTensor(bytes, sourceDtype, tensor.name);
|
|
1230
|
-
const { quantized, scales } = quantizeToInt4PerRowSymmetric(f32, tensor.shape);
|
|
1231
|
-
const scalesBytes = new Uint8Array(scales.buffer, scales.byteOffset, scales.byteLength);
|
|
1232
|
-
const [rows, cols] = resolveOriginalTensorShape(options) ?? tensor.shape;
|
|
1233
|
-
return {
|
|
1234
|
-
tensorData: quantized,
|
|
1235
|
-
companionData: scalesBytes,
|
|
1236
|
-
// outDtype is the LOGICAL dtype the tensor resolves to after dequant at
|
|
1237
|
-
// load time. Storage dtype (INT4) is carried in sourceTransform.sourceDtype.
|
|
1238
|
-
outDtype: 'F16',
|
|
1239
|
-
outLayout: 'row',
|
|
1240
|
-
sourceDtype: String(sourceDtype || '').toUpperCase(),
|
|
1241
|
-
tensorTargetQuant: 'int4_per_row_ple',
|
|
1242
|
-
sourceTransform: {
|
|
1243
|
-
kind: 'litert_axis_dequant',
|
|
1244
|
-
scheme: 'per_axis_affine',
|
|
1245
|
-
sourceDtype: 'INT4',
|
|
1246
|
-
targetDtype: 'F16',
|
|
1247
|
-
storageEncoding: 'offset_binary',
|
|
1248
|
-
scaleSemantics: 'step',
|
|
1249
|
-
storageShape: [rows, cols],
|
|
1250
|
-
quantAxis: 1,
|
|
1251
|
-
// scaleSource gets filled in by the writer with the scales companion's
|
|
1252
|
-
// shard/offset/size after appendTensorBytes().
|
|
1253
|
-
},
|
|
1254
|
-
};
|
|
1255
|
-
}
|
|
1256
|
-
|
|
1257
|
-
export function transformTensorBytes(tensor, rawData, options = {}) {
|
|
1258
|
-
const tensorDataInput = rawData instanceof Uint8Array ? rawData : new Uint8Array(rawData);
|
|
1259
|
-
let tensorData = tensorDataInput;
|
|
1260
|
-
let outDtype = tensor.dtype;
|
|
1261
|
-
let outLayout = null;
|
|
1262
|
-
|
|
1263
|
-
const sourceDtype = String(tensor.dtype).toUpperCase();
|
|
1264
|
-
const targetQuant = normalizeStorageQuant(options.targetQuant ?? options.quantization ?? null);
|
|
1265
|
-
const quantizationInfo = options.quantizationInfo ?? null;
|
|
1266
|
-
const tensorTargetQuant = resolveTensorTargetQuant(
|
|
1267
|
-
tensor,
|
|
1268
|
-
targetQuant,
|
|
1269
|
-
quantizationInfo
|
|
1270
|
-
);
|
|
1271
|
-
const q4kLayout = normalizeQ4KLayout(options.q4kLayout ?? quantizationInfo?.layout);
|
|
1272
|
-
const quantizeEmbeddings = resolveQuantizeEmbeddings(
|
|
1273
|
-
quantizationInfo,
|
|
1274
|
-
options.quantizeEmbeddings
|
|
1275
|
-
);
|
|
1276
|
-
const modulesToNotConvert = normalizeModulesToNotConvert(
|
|
1277
|
-
options.modulesToNotConvert ?? null
|
|
1278
|
-
);
|
|
1279
|
-
const forceQuantizeDecision = (
|
|
1280
|
-
typeof options.forceQuantizeDecision === 'boolean'
|
|
1281
|
-
? options.forceQuantizeDecision
|
|
1282
|
-
: null
|
|
1283
|
-
);
|
|
1284
|
-
|
|
1285
|
-
// Gemma 4 per-layer embeddings use MediaPipe's INT4 per-row symmetric
|
|
1286
|
-
// quantization (verified against gemma-4-E2B-it.litertlm composites:
|
|
1287
|
-
// quantizedDimension=0, one F32 scale per vocab row, zero_point=0). Saves
|
|
1288
|
-
// ~3.5 GB per model vs the default F16 path. Runtime reads via
|
|
1289
|
-
// sourceTransform.kind=litert_axis_dequant with scaleSemantics=step.
|
|
1290
|
-
if (isGemma4PerLayerEmbedTensor(tensor.name) && canInt4QuantizePerRow(tensor, options)) {
|
|
1291
|
-
return buildInt4PerRowPleTransform(tensor, tensorDataInput, sourceDtype, options);
|
|
1292
|
-
}
|
|
1293
|
-
|
|
1294
|
-
if (isCompressedTensorsW4A16CompanionTensor(tensor)) {
|
|
1295
|
-
return {
|
|
1296
|
-
tensorData,
|
|
1297
|
-
outDtype: sourceDtype,
|
|
1298
|
-
outLayout: null,
|
|
1299
|
-
sourceDtype,
|
|
1300
|
-
tensorTargetQuant: null,
|
|
1301
|
-
};
|
|
1302
|
-
}
|
|
1303
|
-
|
|
1304
|
-
if (SOURCE_PACKED_QUANT_DTYPES.has(tensorTargetQuant)) {
|
|
1305
|
-
const sourceQuant = normalizeStorageQuant(sourceDtype);
|
|
1306
|
-
if (sourceQuant === tensorTargetQuant) {
|
|
1307
|
-
const descriptor = cloneSourcePackedStorageDescriptor(tensorTargetQuant);
|
|
1308
|
-
const sourceCompanions = Array.isArray(tensor?.storage?.companions)
|
|
1309
|
-
? tensor.storage.companions.map((companion) => ({ ...companion }))
|
|
1310
|
-
: null;
|
|
1311
|
-
return {
|
|
1312
|
-
tensorData,
|
|
1313
|
-
outDtype: SOURCE_PACKED_MANIFEST_DTYPES[tensorTargetQuant],
|
|
1314
|
-
outLayout: null,
|
|
1315
|
-
sourceDtype,
|
|
1316
|
-
tensorTargetQuant,
|
|
1317
|
-
storage: {
|
|
1318
|
-
...descriptor,
|
|
1319
|
-
...(sourceCompanions ? { companions: sourceCompanions } : {}),
|
|
1320
|
-
},
|
|
1321
|
-
};
|
|
1322
|
-
}
|
|
1323
|
-
const roleQuant = resolveExplicitRoleQuant(tensor, quantizationInfo);
|
|
1324
|
-
const requiresPackedSource = (
|
|
1325
|
-
forceQuantizeDecision
|
|
1326
|
-
?? (
|
|
1327
|
-
roleQuant === tensorTargetQuant
|
|
1328
|
-
|| shouldQuantize(tensor.name, tensor.shape, {
|
|
1329
|
-
quantizeEmbeddings,
|
|
1330
|
-
modulesToNotConvert,
|
|
1331
|
-
role: tensor.role ?? null,
|
|
1332
|
-
})
|
|
1333
|
-
)
|
|
1334
|
-
);
|
|
1335
|
-
if (!requiresPackedSource) {
|
|
1336
|
-
return {
|
|
1337
|
-
tensorData,
|
|
1338
|
-
outDtype: sourceDtype,
|
|
1339
|
-
outLayout: null,
|
|
1340
|
-
sourceDtype,
|
|
1341
|
-
tensorTargetQuant,
|
|
1342
|
-
};
|
|
1343
|
-
}
|
|
1344
|
-
throw new Error(
|
|
1345
|
-
`Cannot materialize ${tensorTargetQuant} for ${tensor.name}: ` +
|
|
1346
|
-
`native import requires source dtype ${SOURCE_PACKED_MANIFEST_DTYPES[tensorTargetQuant]}; ` +
|
|
1347
|
-
'the converter does not re-quantize tensors into this packed format.'
|
|
1348
|
-
);
|
|
1349
|
-
}
|
|
1350
|
-
|
|
1351
|
-
if (tensorTargetQuant === 'q4k') {
|
|
1352
|
-
const sourceQuant = normalizeStorageQuant(sourceDtype);
|
|
1353
|
-
const tensorRole = resolveTensorRole(tensor);
|
|
1354
|
-
const isMatrixLikeShape = Array.isArray(tensor.shape) && tensor.shape.length >= 2;
|
|
1355
|
-
const is2DMatrixShape = Array.isArray(tensor.shape) && tensor.shape.length === 2;
|
|
1356
|
-
const useQ4KRowWise = isMatrixLikeShape
|
|
1357
|
-
&& q4kLayout === 'row'
|
|
1358
|
-
&& (is2DMatrixShape || tensorRole === 'expert');
|
|
1359
|
-
if (sourceQuant === 'q4k') {
|
|
1360
|
-
outDtype = 'Q4_K_M';
|
|
1361
|
-
if (is2DMatrixShape) {
|
|
1362
|
-
outLayout = q4kLayout;
|
|
1363
|
-
}
|
|
1364
|
-
return {
|
|
1365
|
-
tensorData,
|
|
1366
|
-
outDtype,
|
|
1367
|
-
outLayout,
|
|
1368
|
-
sourceDtype,
|
|
1369
|
-
tensorTargetQuant,
|
|
1370
|
-
};
|
|
1371
|
-
}
|
|
1372
|
-
|
|
1373
|
-
const shouldQuantizeTensor = (
|
|
1374
|
-
forceQuantizeDecision ?? shouldQuantize(tensor.name, tensor.shape, {
|
|
1375
|
-
quantizeEmbeddings,
|
|
1376
|
-
modulesToNotConvert,
|
|
1377
|
-
role: tensor.role ?? null,
|
|
1378
|
-
})
|
|
1379
|
-
);
|
|
1380
|
-
if (shouldQuantizeTensor) {
|
|
1381
|
-
const f32Data = toFloat32ForQ4K(tensorData, sourceDtype, tensor.name);
|
|
1382
|
-
const quantized = (
|
|
1383
|
-
is2DMatrixShape
|
|
1384
|
-
? (q4kLayout === 'col'
|
|
1385
|
-
? quantizeToQ4KMColumnWise(f32Data, tensor.shape)
|
|
1386
|
-
: quantizeToQ4KMRowWise(f32Data, tensor.shape))
|
|
1387
|
-
: useQ4KRowWise
|
|
1388
|
-
? quantizeToQ4KMRowWise(f32Data, tensor.shape)
|
|
1389
|
-
: quantizeToQ4KM(f32Data, tensor.shape)
|
|
1390
|
-
);
|
|
1391
|
-
tensorData = quantized.quantized;
|
|
1392
|
-
outDtype = 'Q4_K_M';
|
|
1393
|
-
if (is2DMatrixShape || useQ4KRowWise) {
|
|
1394
|
-
outLayout = q4kLayout;
|
|
1395
|
-
}
|
|
1396
|
-
} else if (sourceDtype === 'BF16') {
|
|
1397
|
-
// BF16 is not a native WebGPU dtype. When quantization is skipped
|
|
1398
|
-
// (e.g. via modulesToNotConvert), convert BF16→F16 so the runtime
|
|
1399
|
-
// can load the tensor without a BF16 dequant shader.
|
|
1400
|
-
const bf16 = new Uint16Array(
|
|
1401
|
-
tensorData.buffer,
|
|
1402
|
-
tensorData.byteOffset,
|
|
1403
|
-
tensorData.byteLength / 2
|
|
1404
|
-
);
|
|
1405
|
-
const f16 = new Uint16Array(bf16.length);
|
|
1406
|
-
for (let j = 0; j < bf16.length; j++) {
|
|
1407
|
-
f16[j] = float32ToFloat16(bf16ToFloat32(bf16[j]));
|
|
1408
|
-
}
|
|
1409
|
-
tensorData = new Uint8Array(f16.buffer, f16.byteOffset, f16.byteLength);
|
|
1410
|
-
outDtype = 'F16';
|
|
1411
|
-
}
|
|
1412
|
-
} else if (tensorTargetQuant === 'f16' && sourceDtype === 'F32') {
|
|
1413
|
-
if (tensorData.byteLength % 4 !== 0) {
|
|
1414
|
-
throw new Error(`Invalid F32 tensor byte length for ${tensor.name}: ${tensorData.byteLength}`);
|
|
1415
|
-
}
|
|
1416
|
-
const f32 = new Float32Array(
|
|
1417
|
-
tensorData.buffer,
|
|
1418
|
-
tensorData.byteOffset,
|
|
1419
|
-
tensorData.byteLength / 4
|
|
1420
|
-
);
|
|
1421
|
-
const f16 = new Uint16Array(f32.length);
|
|
1422
|
-
for (let j = 0; j < f32.length; j++) {
|
|
1423
|
-
f16[j] = float32ToFloat16(f32[j]);
|
|
1424
|
-
}
|
|
1425
|
-
tensorData = new Uint8Array(f16.buffer, f16.byteOffset, f16.byteLength);
|
|
1426
|
-
outDtype = 'F16';
|
|
1427
|
-
} else if (tensorTargetQuant === 'f16' && sourceDtype === 'BF16') {
|
|
1428
|
-
if (tensorData.byteLength % 2 !== 0) {
|
|
1429
|
-
throw new Error(`Invalid BF16 tensor byte length for ${tensor.name}: ${tensorData.byteLength}`);
|
|
1430
|
-
}
|
|
1431
|
-
const bf16 = new Uint16Array(
|
|
1432
|
-
tensorData.buffer,
|
|
1433
|
-
tensorData.byteOffset,
|
|
1434
|
-
tensorData.byteLength / 2
|
|
1435
|
-
);
|
|
1436
|
-
const f16 = new Uint16Array(bf16.length);
|
|
1437
|
-
for (let j = 0; j < bf16.length; j++) {
|
|
1438
|
-
f16[j] = float32ToFloat16(bf16ToFloat32(bf16[j]));
|
|
1439
|
-
}
|
|
1440
|
-
tensorData = new Uint8Array(f16.buffer, f16.byteOffset, f16.byteLength);
|
|
1441
|
-
outDtype = 'F16';
|
|
1442
|
-
} else if (tensorTargetQuant === 'bf16' && sourceDtype === 'F32') {
|
|
1443
|
-
if (tensorData.byteLength % 4 !== 0) {
|
|
1444
|
-
throw new Error(`Invalid F32 tensor byte length for ${tensor.name}: ${tensorData.byteLength}`);
|
|
1445
|
-
}
|
|
1446
|
-
const f32 = new Float32Array(
|
|
1447
|
-
tensorData.buffer,
|
|
1448
|
-
tensorData.byteOffset,
|
|
1449
|
-
tensorData.byteLength / 4
|
|
1450
|
-
);
|
|
1451
|
-
const bf16 = new Uint16Array(f32.length);
|
|
1452
|
-
for (let j = 0; j < f32.length; j++) {
|
|
1453
|
-
bf16[j] = float32ToBFloat16(f32[j]);
|
|
1454
|
-
}
|
|
1455
|
-
tensorData = new Uint8Array(bf16.buffer, bf16.byteOffset, bf16.byteLength);
|
|
1456
|
-
outDtype = 'BF16';
|
|
1457
|
-
} else if (tensorTargetQuant === 'f32' && sourceDtype === 'F16') {
|
|
1458
|
-
if (tensorData.byteLength % 2 !== 0) {
|
|
1459
|
-
throw new Error(`Invalid F16 tensor byte length for ${tensor.name}: ${tensorData.byteLength}`);
|
|
1460
|
-
}
|
|
1461
|
-
const f16 = new Uint16Array(
|
|
1462
|
-
tensorData.buffer,
|
|
1463
|
-
tensorData.byteOffset,
|
|
1464
|
-
tensorData.byteLength / 2
|
|
1465
|
-
);
|
|
1466
|
-
const f32 = new Float32Array(f16.length);
|
|
1467
|
-
for (let j = 0; j < f16.length; j++) {
|
|
1468
|
-
f32[j] = float16ToFloat32(f16[j]);
|
|
1469
|
-
}
|
|
1470
|
-
tensorData = new Uint8Array(f32.buffer, f32.byteOffset, f32.byteLength);
|
|
1471
|
-
outDtype = 'F32';
|
|
1472
|
-
} else if (tensorTargetQuant === 'f32' && sourceDtype === 'BF16') {
|
|
1473
|
-
if (tensorData.byteLength % 2 !== 0) {
|
|
1474
|
-
throw new Error(`Invalid BF16 tensor byte length for ${tensor.name}: ${tensorData.byteLength}`);
|
|
1475
|
-
}
|
|
1476
|
-
const bf16 = new Uint16Array(
|
|
1477
|
-
tensorData.buffer,
|
|
1478
|
-
tensorData.byteOffset,
|
|
1479
|
-
tensorData.byteLength / 2
|
|
1480
|
-
);
|
|
1481
|
-
const f32 = new Float32Array(bf16.length);
|
|
1482
|
-
for (let j = 0; j < bf16.length; j++) {
|
|
1483
|
-
f32[j] = bf16ToFloat32(bf16[j]);
|
|
1484
|
-
}
|
|
1485
|
-
tensorData = new Uint8Array(f32.buffer, f32.byteOffset, f32.byteLength);
|
|
1486
|
-
outDtype = 'F32';
|
|
1487
|
-
}
|
|
1488
|
-
|
|
1489
|
-
return {
|
|
1490
|
-
tensorData,
|
|
1491
|
-
outDtype,
|
|
1492
|
-
outLayout,
|
|
1493
|
-
sourceDtype,
|
|
1494
|
-
tensorTargetQuant,
|
|
1495
|
-
};
|
|
1496
|
-
}
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
export function extractArchitecture(config, ggufConfig) {
|
|
1500
|
-
const firstNumber = (...values) => {
|
|
1501
|
-
for (const value of values) {
|
|
1502
|
-
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
1503
|
-
return value;
|
|
1504
|
-
}
|
|
1505
|
-
}
|
|
1506
|
-
return null;
|
|
1507
|
-
};
|
|
1508
|
-
|
|
1509
|
-
const requireNumber = (value, label) => {
|
|
1510
|
-
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
|
1511
|
-
throw new Error(`Missing ${label} in model config`);
|
|
1512
|
-
}
|
|
1513
|
-
return value;
|
|
1514
|
-
};
|
|
1515
|
-
|
|
1516
|
-
const normalizeLinearNormMode = (value, sharedFlag = null) => {
|
|
1517
|
-
if (typeof value === 'string') {
|
|
1518
|
-
const normalized = value.trim().toLowerCase();
|
|
1519
|
-
if (normalized === 'shared') return 'shared';
|
|
1520
|
-
if (normalized === 'per_head' || normalized === 'per-head' || normalized === 'perhead') {
|
|
1521
|
-
return 'per_head';
|
|
1522
|
-
}
|
|
1523
|
-
throw new Error(
|
|
1524
|
-
`Unsupported linear_norm_mode="${value}" in model config. Supported values: "shared", "per_head".`
|
|
1525
|
-
);
|
|
1526
|
-
}
|
|
1527
|
-
if (typeof sharedFlag === 'boolean') {
|
|
1528
|
-
return sharedFlag ? 'shared' : 'per_head';
|
|
1529
|
-
}
|
|
1530
|
-
return undefined;
|
|
1531
|
-
};
|
|
1532
|
-
|
|
1533
|
-
// Try HuggingFace config first
|
|
1534
|
-
if (config && Object.keys(config).length > 0) {
|
|
1535
|
-
const textConfig = getNestedTextConfig(config);
|
|
1536
|
-
const fromConfig = (...keys) => {
|
|
1537
|
-
const values = [];
|
|
1538
|
-
for (const key of keys) {
|
|
1539
|
-
values.push(config[key]);
|
|
1540
|
-
}
|
|
1541
|
-
for (const key of keys) {
|
|
1542
|
-
values.push(textConfig?.[key]);
|
|
1543
|
-
}
|
|
1544
|
-
return firstNumber(...values);
|
|
1545
|
-
};
|
|
1546
|
-
const fromConfigValue = (...keys) => {
|
|
1547
|
-
for (const key of keys) {
|
|
1548
|
-
if (config[key] !== undefined) return config[key];
|
|
1549
|
-
}
|
|
1550
|
-
for (const key of keys) {
|
|
1551
|
-
if (textConfig?.[key] !== undefined) return textConfig[key];
|
|
1552
|
-
}
|
|
1553
|
-
return undefined;
|
|
1554
|
-
};
|
|
1555
|
-
const numLayers = requireNumber(
|
|
1556
|
-
fromConfig('num_hidden_layers', 'n_layer', 'num_layers'),
|
|
1557
|
-
'num_hidden_layers'
|
|
1558
|
-
);
|
|
1559
|
-
const hiddenSize = requireNumber(
|
|
1560
|
-
fromConfig('hidden_size', 'n_embd', 'embedding_size'),
|
|
1561
|
-
'hidden_size'
|
|
1562
|
-
);
|
|
1563
|
-
const intermediateSize = requireNumber(
|
|
1564
|
-
fromConfig('intermediate_size', 'n_inner', 'ffn_dim'),
|
|
1565
|
-
'intermediate_size'
|
|
1566
|
-
);
|
|
1567
|
-
const numHeads = requireNumber(
|
|
1568
|
-
fromConfig('num_attention_heads', 'n_head', 'attention_heads'),
|
|
1569
|
-
'num_attention_heads'
|
|
1570
|
-
);
|
|
1571
|
-
const numKVHeads = fromConfig('num_key_value_heads', 'num_kv_heads') ?? numHeads;
|
|
1572
|
-
const numGlobalKVHeads = fromConfig('num_global_key_value_heads', 'num_global_kv_heads');
|
|
1573
|
-
const headDimFromConfig = fromConfig('head_dim') ?? Math.floor(hiddenSize / numHeads);
|
|
1574
|
-
const vocabSize = requireNumber(
|
|
1575
|
-
fromConfig('vocab_size', 'n_vocab'),
|
|
1576
|
-
'vocab_size'
|
|
1577
|
-
);
|
|
1578
|
-
const maxSeqLen = requireNumber(
|
|
1579
|
-
fromConfig('max_position_embeddings', 'n_positions', 'max_seq_len', 'max_length'),
|
|
1580
|
-
'max_position_embeddings'
|
|
1581
|
-
);
|
|
1582
|
-
const ropeTheta = fromConfig('rope_theta') ?? undefined;
|
|
1583
|
-
const linearNumKeyHeads = fromConfig('linear_num_key_heads');
|
|
1584
|
-
const linearNumValueHeads = fromConfig('linear_num_value_heads');
|
|
1585
|
-
const linearKeyHeadDim = fromConfig('linear_key_head_dim');
|
|
1586
|
-
const linearValueHeadDim = fromConfig('linear_value_head_dim');
|
|
1587
|
-
const linearConvKernelDim = fromConfig('linear_conv_kernel_dim');
|
|
1588
|
-
const hiddenSizePerLayerInput = fromConfig('hidden_size_per_layer_input');
|
|
1589
|
-
const vocabSizePerLayerInput = fromConfig('vocab_size_per_layer_input');
|
|
1590
|
-
const globalHeadDim = fromConfig('global_head_dim');
|
|
1591
|
-
const numKvSharedLayers = fromConfig('num_kv_shared_layers');
|
|
1592
|
-
const linearNormModeConfigured = normalizeLinearNormMode(
|
|
1593
|
-
fromConfigValue('linear_norm_mode'),
|
|
1594
|
-
fromConfigValue('linear_norm_shared')
|
|
1595
|
-
);
|
|
1596
|
-
const modelType = String(fromConfigValue('model_type') ?? '').trim().toLowerCase();
|
|
1597
|
-
const rawLayerTypes = fromConfigValue('layer_types');
|
|
1598
|
-
const layerTypes = Array.isArray(rawLayerTypes) ? rawLayerTypes : null;
|
|
1599
|
-
const hasLinearLayers = Array.isArray(layerTypes)
|
|
1600
|
-
&& layerTypes.some((entry) => {
|
|
1601
|
-
const normalized = String(entry ?? '').trim().toLowerCase();
|
|
1602
|
-
return normalized === 'linear_attention'
|
|
1603
|
-
|| normalized === 'linear'
|
|
1604
|
-
|| normalized === 'gated_delta'
|
|
1605
|
-
|| normalized === 'gated_delta_net';
|
|
1606
|
-
});
|
|
1607
|
-
const linearNormMode = linearNormModeConfigured
|
|
1608
|
-
?? ((hasLinearLayers && modelType.startsWith('qwen')) ? 'shared' : undefined);
|
|
1609
|
-
|
|
1610
|
-
return {
|
|
1611
|
-
numLayers,
|
|
1612
|
-
hiddenSize,
|
|
1613
|
-
intermediateSize,
|
|
1614
|
-
numAttentionHeads: numHeads,
|
|
1615
|
-
numKeyValueHeads: numKVHeads,
|
|
1616
|
-
numGlobalKeyValueHeads: numGlobalKVHeads ?? undefined,
|
|
1617
|
-
headDim: headDimFromConfig,
|
|
1618
|
-
vocabSize,
|
|
1619
|
-
maxSeqLen,
|
|
1620
|
-
ropeTheta,
|
|
1621
|
-
linearNumKeyHeads,
|
|
1622
|
-
linearNumValueHeads,
|
|
1623
|
-
linearKeyHeadDim,
|
|
1624
|
-
linearValueHeadDim,
|
|
1625
|
-
linearConvKernelDim,
|
|
1626
|
-
hiddenSizePerLayerInput,
|
|
1627
|
-
vocabSizePerLayerInput,
|
|
1628
|
-
globalHeadDim,
|
|
1629
|
-
numKvSharedLayers,
|
|
1630
|
-
linearNormMode,
|
|
1631
|
-
};
|
|
1632
|
-
}
|
|
1633
|
-
|
|
1634
|
-
// GGUF config
|
|
1635
|
-
if (ggufConfig) {
|
|
1636
|
-
const c = ggufConfig;
|
|
1637
|
-
const numLayers = requireNumber(
|
|
1638
|
-
firstNumber(c.blockCount, c.block_count),
|
|
1639
|
-
'blockCount'
|
|
1640
|
-
);
|
|
1641
|
-
const hiddenSize = requireNumber(
|
|
1642
|
-
firstNumber(c.embeddingLength, c.embedding_length),
|
|
1643
|
-
'embeddingLength'
|
|
1644
|
-
);
|
|
1645
|
-
const intermediateSize = requireNumber(
|
|
1646
|
-
firstNumber(c.feedForwardLength, c.feed_forward_length),
|
|
1647
|
-
'feedForwardLength'
|
|
1648
|
-
);
|
|
1649
|
-
const numHeads = requireNumber(
|
|
1650
|
-
firstNumber(c.attentionHeadCount, c.attention_head_count),
|
|
1651
|
-
'attentionHeadCount'
|
|
1652
|
-
);
|
|
1653
|
-
const numKVHeads = firstNumber(c.attentionHeadCountKV, c.attention_head_count_kv) ?? numHeads;
|
|
1654
|
-
const vocabSize = requireNumber(
|
|
1655
|
-
firstNumber(c.vocabSize, c.vocab_size),
|
|
1656
|
-
'vocabSize'
|
|
1657
|
-
);
|
|
1658
|
-
const maxSeqLen = requireNumber(
|
|
1659
|
-
firstNumber(c.contextLength, c.context_length),
|
|
1660
|
-
'contextLength'
|
|
1661
|
-
);
|
|
1662
|
-
|
|
1663
|
-
// Gemma 4-specific fields (optional — undefined on non-Gemma-4 GGUFs).
|
|
1664
|
-
// key_length is per-head; pick the larger of (key_length, key_length_swa)
|
|
1665
|
-
// for globalHeadDim and the smaller for headDim. Matches the mixed-geometry
|
|
1666
|
-
// KV cache expected by src/inference/pipelines/text/layer.js.
|
|
1667
|
-
const keyLen = firstNumber(c.attentionKeyLength);
|
|
1668
|
-
const keyLenSwa = firstNumber(c.attentionKeyLengthSwa);
|
|
1669
|
-
const headDimFromGguf = keyLenSwa != null && keyLen != null
|
|
1670
|
-
? Math.min(keyLen, keyLenSwa)
|
|
1671
|
-
: (keyLenSwa ?? Math.floor(hiddenSize / numHeads));
|
|
1672
|
-
const globalHeadDim = (keyLen != null && keyLenSwa != null && keyLen !== keyLenSwa)
|
|
1673
|
-
? Math.max(keyLen, keyLenSwa)
|
|
1674
|
-
: undefined;
|
|
1675
|
-
const numKvSharedLayers = firstNumber(c.numKvSharedLayers);
|
|
1676
|
-
const hiddenSizePerLayerInput = firstNumber(c.hiddenSizePerLayerInput);
|
|
1677
|
-
const vocabSizePerLayerInput = hiddenSizePerLayerInput != null ? vocabSize : undefined;
|
|
1678
|
-
|
|
1679
|
-
return {
|
|
1680
|
-
numLayers,
|
|
1681
|
-
hiddenSize,
|
|
1682
|
-
intermediateSize,
|
|
1683
|
-
numAttentionHeads: numHeads,
|
|
1684
|
-
numKeyValueHeads: numKVHeads,
|
|
1685
|
-
headDim: headDimFromGguf,
|
|
1686
|
-
vocabSize,
|
|
1687
|
-
maxSeqLen,
|
|
1688
|
-
...(globalHeadDim != null ? { globalHeadDim } : {}),
|
|
1689
|
-
...(numKvSharedLayers != null ? { numKvSharedLayers } : {}),
|
|
1690
|
-
...(hiddenSizePerLayerInput != null ? { hiddenSizePerLayerInput } : {}),
|
|
1691
|
-
...(vocabSizePerLayerInput != null ? { vocabSizePerLayerInput } : {}),
|
|
1692
|
-
};
|
|
1693
|
-
}
|
|
1694
|
-
|
|
1695
|
-
throw new Error('Missing model config: cannot extract architecture');
|
|
1696
|
-
}
|
|
1697
|
-
|
|
1698
|
-
function getNestedTextConfig(config) {
|
|
1699
|
-
if (!config || typeof config !== 'object' || Array.isArray(config)) {
|
|
1700
|
-
return null;
|
|
1701
|
-
}
|
|
1702
|
-
if (config.text_config && typeof config.text_config === 'object' && !Array.isArray(config.text_config)) {
|
|
1703
|
-
return config.text_config;
|
|
1704
|
-
}
|
|
1705
|
-
if (config.language_config && typeof config.language_config === 'object' && !Array.isArray(config.language_config)) {
|
|
1706
|
-
return config.language_config;
|
|
1707
|
-
}
|
|
1708
|
-
return null;
|
|
1709
|
-
}
|
|
1710
|
-
|
|
1711
|
-
function resolveGemma4TextConfig(rawConfig) {
|
|
1712
|
-
const textConfig = getNestedTextConfig(rawConfig);
|
|
1713
|
-
const modelType = String(textConfig?.model_type ?? rawConfig?.model_type ?? '').trim().toLowerCase();
|
|
1714
|
-
if (
|
|
1715
|
-
modelType !== 'gemma4'
|
|
1716
|
-
&& modelType !== 'gemma4_text'
|
|
1717
|
-
&& modelType !== 'gemma4_unified'
|
|
1718
|
-
&& modelType !== 'gemma4_unified_text'
|
|
1719
|
-
) {
|
|
1720
|
-
return null;
|
|
1721
|
-
}
|
|
1722
|
-
return textConfig ?? rawConfig ?? null;
|
|
1723
|
-
}
|
|
1724
|
-
|
|
1725
|
-
function resolveDiffusionGemmaConfig(rawConfig, resolvedModelType = null) {
|
|
1726
|
-
const textConfig = getNestedTextConfig(rawConfig);
|
|
1727
|
-
const modelType = String(
|
|
1728
|
-
resolvedModelType
|
|
1729
|
-
?? rawConfig?.model_type
|
|
1730
|
-
?? textConfig?.model_type
|
|
1731
|
-
?? ''
|
|
1732
|
-
).trim().toLowerCase();
|
|
1733
|
-
const textModelType = String(textConfig?.model_type ?? '').trim().toLowerCase();
|
|
1734
|
-
if (
|
|
1735
|
-
modelType !== 'diffusion_gemma'
|
|
1736
|
-
&& modelType !== 'diffusion_gemma_text'
|
|
1737
|
-
&& textModelType !== 'diffusion_gemma'
|
|
1738
|
-
&& textModelType !== 'diffusion_gemma_text'
|
|
1739
|
-
) {
|
|
1740
|
-
return null;
|
|
1741
|
-
}
|
|
1742
|
-
return textConfig ?? rawConfig ?? {};
|
|
1743
|
-
}
|
|
1744
|
-
|
|
1745
|
-
function collectGemma4UnsupportedTensorFlags(tensors) {
|
|
1746
|
-
const names = Array.isArray(tensors) ? tensors.map((tensor) => normalizeTensorName(tensor).toLowerCase()) : [];
|
|
1747
|
-
const flags = [];
|
|
1748
|
-
if (names.some((name) => name.includes('.experts.gate_up_proj'))) {
|
|
1749
|
-
flags.push('experts.gate_up_proj');
|
|
1750
|
-
}
|
|
1751
|
-
if (names.some((name) => name.includes('.experts.down_proj'))) {
|
|
1752
|
-
flags.push('experts.down_proj');
|
|
1753
|
-
}
|
|
1754
|
-
if (names.some((name) => name.includes('.router.per_expert_scale'))) {
|
|
1755
|
-
flags.push('router.per_expert_scale');
|
|
1756
|
-
}
|
|
1757
|
-
if (names.some((name) => name.includes('.router.scale'))) {
|
|
1758
|
-
flags.push('router.scale');
|
|
1759
|
-
}
|
|
1760
|
-
if (names.some((name) => name.includes('.post_feedforward_layernorm_1.'))) {
|
|
1761
|
-
flags.push('post_feedforward_layernorm_1');
|
|
1762
|
-
}
|
|
1763
|
-
if (names.some((name) => name.includes('.post_feedforward_layernorm_2.'))) {
|
|
1764
|
-
flags.push('post_feedforward_layernorm_2');
|
|
1765
|
-
}
|
|
1766
|
-
if (names.some((name) => name.includes('.pre_feedforward_layernorm_2.'))) {
|
|
1767
|
-
flags.push('pre_feedforward_layernorm_2');
|
|
1768
|
-
}
|
|
1769
|
-
return flags;
|
|
1770
|
-
}
|
|
1771
|
-
|
|
1772
|
-
function assertSupportedGemma4Conversion(model, tensors, modelId) {
|
|
1773
|
-
const rawConfig = model?.config ?? null;
|
|
1774
|
-
const textConfig = resolveGemma4TextConfig(rawConfig);
|
|
1775
|
-
if (!textConfig) return;
|
|
1776
|
-
|
|
1777
|
-
const hiddenSizePerLayerInput = Number(textConfig.hidden_size_per_layer_input ?? 0);
|
|
1778
|
-
if (Number.isFinite(hiddenSizePerLayerInput) && hiddenSizePerLayerInput > 0) {
|
|
1779
|
-
const names = Array.isArray(tensors)
|
|
1780
|
-
? tensors.map((tensor) => String(tensor?.name ?? '').trim())
|
|
1781
|
-
: [];
|
|
1782
|
-
const requiredNames = [
|
|
1783
|
-
'embed_tokens_per_layer.weight',
|
|
1784
|
-
'per_layer_input_gate.weight',
|
|
1785
|
-
'per_layer_projection.weight',
|
|
1786
|
-
'post_per_layer_input_norm.weight',
|
|
1787
|
-
'per_layer_model_projection.weight',
|
|
1788
|
-
'per_layer_projection_norm.weight',
|
|
1789
|
-
];
|
|
1790
|
-
const missing = requiredNames.filter((suffix) => !names.some((name) => name.endsWith(suffix)));
|
|
1791
|
-
if (missing.length > 0) {
|
|
1792
|
-
throw new Error(
|
|
1793
|
-
`Gemma 4 model "${modelId}" declares hidden_size_per_layer_input=${hiddenSizePerLayerInput}, ` +
|
|
1794
|
-
`but the checkpoint is missing required per-layer input tensors: ${missing.join(', ')}.`
|
|
1795
|
-
);
|
|
1796
|
-
}
|
|
1797
|
-
}
|
|
1798
|
-
|
|
1799
|
-
if (textConfig.enable_moe_block !== true) {
|
|
1800
|
-
return;
|
|
1801
|
-
}
|
|
1802
|
-
|
|
1803
|
-
const unsupportedFlags = collectGemma4UnsupportedTensorFlags(tensors);
|
|
1804
|
-
if (unsupportedFlags.length === 0 && !modelHasMoETensors({ tensors })) {
|
|
1805
|
-
return;
|
|
1806
|
-
}
|
|
1807
|
-
|
|
1808
|
-
throw new Error(
|
|
1809
|
-
`Gemma 4 model "${modelId}" is not supported yet: Gemma 4 MoE decoder blocks require ` +
|
|
1810
|
-
'Gemma-specific router scaling and dual dense+MoE FFN execution, but current Doppler MoE runtime ' +
|
|
1811
|
-
'only supports Mixtral/GPT-OSS semantics. ' +
|
|
1812
|
-
`Detected: ${unsupportedFlags.length > 0 ? unsupportedFlags.join(', ') : 'Gemma 4 MoE tensors'}.`
|
|
1813
|
-
);
|
|
1814
|
-
}
|
|
1815
|
-
|
|
1816
|
-
function readDiffusionGemmaPositiveInteger(value, label, modelId) {
|
|
1817
|
-
if (!Number.isInteger(value) || value <= 0) {
|
|
1818
|
-
throw new Error(`DiffusionGemma model "${modelId}" requires ${label} to be a positive integer.`);
|
|
1819
|
-
}
|
|
1820
|
-
return value;
|
|
1821
|
-
}
|
|
1822
|
-
|
|
1823
|
-
function readDiffusionGemmaNonNegativeNumber(value, label, modelId) {
|
|
1824
|
-
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
|
1825
|
-
throw new Error(`DiffusionGemma model "${modelId}" requires ${label} to be a non-negative finite number.`);
|
|
1826
|
-
}
|
|
1827
|
-
return value;
|
|
1828
|
-
}
|
|
1829
|
-
|
|
1830
|
-
function readDiffusionGemmaEntropyBound(generationConfig, modelId) {
|
|
1831
|
-
if (generationConfig && Object.hasOwn(generationConfig, 'entropy_bound')) {
|
|
1832
|
-
return readDiffusionGemmaNonNegativeNumber(
|
|
1833
|
-
generationConfig.entropy_bound,
|
|
1834
|
-
'generation_config.entropy_bound',
|
|
1835
|
-
modelId
|
|
1836
|
-
);
|
|
1837
|
-
}
|
|
1838
|
-
return readDiffusionGemmaNonNegativeNumber(
|
|
1839
|
-
generationConfig?.sampler_config?.entropy_bound,
|
|
1840
|
-
'generation_config.sampler_config.entropy_bound',
|
|
1841
|
-
modelId
|
|
1842
|
-
);
|
|
1843
|
-
}
|
|
1844
|
-
|
|
1845
|
-
function readDiffusionGemmaNullableTokenId(value, label, modelId) {
|
|
1846
|
-
if (value == null) return null;
|
|
1847
|
-
if (!Number.isInteger(value) || value < 0) {
|
|
1848
|
-
throw new Error(`DiffusionGemma model "${modelId}" requires ${label} to be null or a non-negative integer.`);
|
|
1849
|
-
}
|
|
1850
|
-
return value;
|
|
1851
|
-
}
|
|
1852
|
-
|
|
1853
|
-
function readDiffusionGemmaTokenId(value, label, modelId) {
|
|
1854
|
-
if (!Number.isInteger(value) || value < 0) {
|
|
1855
|
-
throw new Error(`DiffusionGemma model "${modelId}" requires ${label} to be a non-negative integer.`);
|
|
1856
|
-
}
|
|
1857
|
-
return value;
|
|
1858
|
-
}
|
|
1859
|
-
|
|
1860
|
-
function readDiffusionGemmaEosTokenIds(rawConfig, generationConfig, modelId) {
|
|
1861
|
-
const eos = resolveTokenizerIds(
|
|
1862
|
-
generationConfig?.eos_token_id
|
|
1863
|
-
?? generationConfig?.eos_token_ids
|
|
1864
|
-
?? rawConfig?.eos_token_id
|
|
1865
|
-
?? rawConfig?.eos_token_ids
|
|
1866
|
-
);
|
|
1867
|
-
if (!Array.isArray(eos) || eos.length === 0) {
|
|
1868
|
-
throw new Error(`DiffusionGemma model "${modelId}" requires generation_config.eos_token_id.`);
|
|
1869
|
-
}
|
|
1870
|
-
return eos;
|
|
1871
|
-
}
|
|
1872
|
-
|
|
1873
|
-
function resolveDiffusionGemmaInferenceContract(rawConfig, generationConfig, modelId) {
|
|
1874
|
-
const canvasLength = readDiffusionGemmaPositiveInteger(
|
|
1875
|
-
rawConfig?.canvas_length,
|
|
1876
|
-
'config.canvas_length',
|
|
1877
|
-
modelId
|
|
1878
|
-
);
|
|
1879
|
-
const maxDenoisingSteps = readDiffusionGemmaPositiveInteger(
|
|
1880
|
-
generationConfig?.max_denoising_steps,
|
|
1881
|
-
'generation_config.max_denoising_steps',
|
|
1882
|
-
modelId
|
|
1883
|
-
);
|
|
1884
|
-
const maxNewTokens = readDiffusionGemmaPositiveInteger(
|
|
1885
|
-
generationConfig?.max_new_tokens,
|
|
1886
|
-
'generation_config.max_new_tokens',
|
|
1887
|
-
modelId
|
|
1888
|
-
);
|
|
1889
|
-
const stabilityThreshold = readDiffusionGemmaPositiveInteger(
|
|
1890
|
-
generationConfig?.stability_threshold,
|
|
1891
|
-
'generation_config.stability_threshold',
|
|
1892
|
-
modelId
|
|
1893
|
-
);
|
|
1894
|
-
const padTokenId = readDiffusionGemmaTokenId(
|
|
1895
|
-
generationConfig?.pad_token_id ?? rawConfig?.pad_token_id,
|
|
1896
|
-
'generation_config.pad_token_id',
|
|
1897
|
-
modelId
|
|
1898
|
-
);
|
|
1899
|
-
const tMin = readDiffusionGemmaNonNegativeNumber(
|
|
1900
|
-
generationConfig?.t_min,
|
|
1901
|
-
'generation_config.t_min',
|
|
1902
|
-
modelId
|
|
1903
|
-
);
|
|
1904
|
-
const tMax = readDiffusionGemmaNonNegativeNumber(
|
|
1905
|
-
generationConfig?.t_max,
|
|
1906
|
-
'generation_config.t_max',
|
|
1907
|
-
modelId
|
|
1908
|
-
);
|
|
1909
|
-
if (tMax < tMin) {
|
|
1910
|
-
throw new Error(`DiffusionGemma model "${modelId}" requires generation_config.t_max >= t_min.`);
|
|
1911
|
-
}
|
|
1912
|
-
return {
|
|
1913
|
-
canvasLength,
|
|
1914
|
-
maxDenoisingSteps,
|
|
1915
|
-
maxNewTokens,
|
|
1916
|
-
tMin,
|
|
1917
|
-
tMax,
|
|
1918
|
-
entropyBound: readDiffusionGemmaEntropyBound(generationConfig, modelId),
|
|
1919
|
-
confidenceThreshold: readDiffusionGemmaNonNegativeNumber(
|
|
1920
|
-
generationConfig?.confidence_threshold,
|
|
1921
|
-
'generation_config.confidence_threshold',
|
|
1922
|
-
modelId
|
|
1923
|
-
),
|
|
1924
|
-
stabilityThreshold,
|
|
1925
|
-
padTokenId,
|
|
1926
|
-
eosTokenIds: readDiffusionGemmaEosTokenIds(rawConfig, generationConfig, modelId),
|
|
1927
|
-
boiTokenId: readDiffusionGemmaNullableTokenId(rawConfig?.boi_token_id, 'config.boi_token_id', modelId),
|
|
1928
|
-
eoiTokenId: readDiffusionGemmaNullableTokenId(rawConfig?.eoi_token_id, 'config.eoi_token_id', modelId),
|
|
1929
|
-
imageTokenId: readDiffusionGemmaNullableTokenId(rawConfig?.image_token_id, 'config.image_token_id', modelId),
|
|
1930
|
-
selfConditioning: true,
|
|
1931
|
-
decoderCacheMode: 'encoder_kv_readonly_canvas_concat',
|
|
1932
|
-
router: {
|
|
1933
|
-
scaleHiddenStates: true,
|
|
1934
|
-
normalizeTopK: true,
|
|
1935
|
-
perExpertScale: true,
|
|
1936
|
-
},
|
|
1937
|
-
};
|
|
1938
|
-
}
|
|
1939
|
-
|
|
1940
|
-
function applyDiffusionGemmaInferenceContract(inference, rawConfig, generationConfig, modelId, resolvedModelType) {
|
|
1941
|
-
if (!resolveDiffusionGemmaConfig(rawConfig, resolvedModelType)) {
|
|
1942
|
-
return inference;
|
|
1943
|
-
}
|
|
1944
|
-
if (inference?.diffusionGemma && typeof inference.diffusionGemma === 'object') {
|
|
1945
|
-
return inference;
|
|
1946
|
-
}
|
|
1947
|
-
return {
|
|
1948
|
-
...inference,
|
|
1949
|
-
diffusionGemma: resolveDiffusionGemmaInferenceContract(rawConfig, generationConfig, modelId),
|
|
1950
|
-
};
|
|
1951
|
-
}
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
export function buildTensorMap(tensors, shardSize) {
|
|
1955
|
-
if (!shardSize || shardSize <= 0) {
|
|
1956
|
-
throw new Error('Missing shard size for tensor map');
|
|
1957
|
-
}
|
|
1958
|
-
const tensorMap = {};
|
|
343
|
+
const tensorMap = {};
|
|
1959
344
|
|
|
1960
345
|
let globalOffset = 0;
|
|
1961
346
|
for (const tensor of tensors) {
|
|
@@ -2005,393 +390,6 @@ export function buildTensorMap(tensors, shardSize) {
|
|
|
2005
390
|
return tensorMap;
|
|
2006
391
|
}
|
|
2007
392
|
|
|
2008
|
-
export function resolveConvertedAt(value) {
|
|
2009
|
-
if (value === undefined || value === null || value === '') {
|
|
2010
|
-
return new Date().toISOString();
|
|
2011
|
-
}
|
|
2012
|
-
if (typeof value !== 'string') {
|
|
2013
|
-
throw new Error('manifest convertedAt must be a string when provided.');
|
|
2014
|
-
}
|
|
2015
|
-
const parsed = Date.parse(value);
|
|
2016
|
-
if (Number.isNaN(parsed)) {
|
|
2017
|
-
throw new Error(`Invalid manifest convertedAt timestamp: "${value}"`);
|
|
2018
|
-
}
|
|
2019
|
-
return new Date(parsed).toISOString();
|
|
2020
|
-
}
|
|
2021
|
-
|
|
2022
|
-
export function resolveManifestMultimodalConfig(rawConfig, manifestConfig = null) {
|
|
2023
|
-
const explicitVisionConfig = manifestConfig?.visionConfig;
|
|
2024
|
-
const explicitAudioConfig = manifestConfig?.audioConfig;
|
|
2025
|
-
const visionConfig = explicitVisionConfig ?? rawConfig?.vision_config ?? null;
|
|
2026
|
-
const audioConfig = explicitAudioConfig ?? rawConfig?.audio_config ?? null;
|
|
2027
|
-
return {
|
|
2028
|
-
vision_config: visionConfig ? cloneJsonValue(visionConfig) : null,
|
|
2029
|
-
audio_config: audioConfig ? cloneJsonValue(audioConfig) : null,
|
|
2030
|
-
};
|
|
2031
|
-
}
|
|
2032
|
-
|
|
2033
|
-
function isPlainRecord(value) {
|
|
2034
|
-
return value != null && typeof value === 'object' && !Array.isArray(value);
|
|
2035
|
-
}
|
|
2036
|
-
|
|
2037
|
-
function stripUndefined(value) {
|
|
2038
|
-
if (Array.isArray(value)) {
|
|
2039
|
-
return value.map(stripUndefined);
|
|
2040
|
-
}
|
|
2041
|
-
if (!isPlainRecord(value)) {
|
|
2042
|
-
return value;
|
|
2043
|
-
}
|
|
2044
|
-
const out = {};
|
|
2045
|
-
for (const key of Object.keys(value).sort()) {
|
|
2046
|
-
const item = value[key];
|
|
2047
|
-
if (item !== undefined) {
|
|
2048
|
-
out[key] = stripUndefined(item);
|
|
2049
|
-
}
|
|
2050
|
-
}
|
|
2051
|
-
return out;
|
|
2052
|
-
}
|
|
2053
|
-
|
|
2054
|
-
function canonicalJson(value) {
|
|
2055
|
-
return JSON.stringify(stripUndefined(value));
|
|
2056
|
-
}
|
|
2057
|
-
|
|
2058
|
-
function normalizeDigest(value, label) {
|
|
2059
|
-
const digest = typeof value === 'string' ? value.trim() : '';
|
|
2060
|
-
if (!digest) {
|
|
2061
|
-
throw new Error(`Missing ${label} digest`);
|
|
2062
|
-
}
|
|
2063
|
-
return digest.startsWith('sha256:') ? digest : `sha256:${digest}`;
|
|
2064
|
-
}
|
|
2065
|
-
|
|
2066
|
-
function digestSuffix(value, length = 12) {
|
|
2067
|
-
return String(value || '').replace(/^sha256:/, '').slice(0, length);
|
|
2068
|
-
}
|
|
2069
|
-
|
|
2070
|
-
async function hashArtifactValue(hashString, value, label) {
|
|
2071
|
-
if (typeof hashString !== 'function') {
|
|
2072
|
-
return null;
|
|
2073
|
-
}
|
|
2074
|
-
const digest = await hashString(canonicalJson(value));
|
|
2075
|
-
return normalizeDigest(digest, label);
|
|
2076
|
-
}
|
|
2077
|
-
|
|
2078
|
-
function resolveArtifactSourceFormat(options) {
|
|
2079
|
-
const explicit = typeof options?.sourceFormat === 'string' ? options.sourceFormat.trim() : '';
|
|
2080
|
-
if (explicit) return explicit;
|
|
2081
|
-
const sourcePath = typeof options?.sourcePath === 'string' ? options.sourcePath.trim().toLowerCase() : '';
|
|
2082
|
-
if (sourcePath.endsWith('.gguf')) return 'gguf';
|
|
2083
|
-
if (sourcePath.endsWith('.tflite')) return 'tflite';
|
|
2084
|
-
if (sourcePath.endsWith('.task')) return 'task';
|
|
2085
|
-
if (sourcePath.endsWith('.litertlm')) return 'litertlm';
|
|
2086
|
-
return 'safetensors';
|
|
2087
|
-
}
|
|
2088
|
-
|
|
2089
|
-
function resolveSourceCheckpointIdentity(explicit, options) {
|
|
2090
|
-
const sourceRepo = typeof explicit?.sourceRepo === 'string' && explicit.sourceRepo.trim()
|
|
2091
|
-
? explicit.sourceRepo.trim()
|
|
2092
|
-
: null;
|
|
2093
|
-
const sourceRevision = typeof explicit?.sourceRevision === 'string' && explicit.sourceRevision.trim()
|
|
2094
|
-
? explicit.sourceRevision.trim()
|
|
2095
|
-
: null;
|
|
2096
|
-
const sourcePath = typeof options?.sourcePath === 'string' && options.sourcePath.trim()
|
|
2097
|
-
? options.sourcePath.trim()
|
|
2098
|
-
: null;
|
|
2099
|
-
const source = typeof options?.source === 'string' && options.source.trim()
|
|
2100
|
-
? options.source.trim()
|
|
2101
|
-
: null;
|
|
2102
|
-
const sourceCheckpointId = typeof explicit?.sourceCheckpointId === 'string' && explicit.sourceCheckpointId.trim()
|
|
2103
|
-
? explicit.sourceCheckpointId.trim()
|
|
2104
|
-
: (
|
|
2105
|
-
sourceRepo && sourceRevision
|
|
2106
|
-
? `${sourceRepo}@${sourceRevision}`
|
|
2107
|
-
: (sourceRepo ?? source ?? sourcePath ?? null)
|
|
2108
|
-
);
|
|
2109
|
-
return {
|
|
2110
|
-
sourceCheckpointId,
|
|
2111
|
-
sourceRepo,
|
|
2112
|
-
sourceRevision,
|
|
2113
|
-
};
|
|
2114
|
-
}
|
|
2115
|
-
|
|
2116
|
-
function inferArtifactModalitySet(modelType, tensorLocations, converterConfig) {
|
|
2117
|
-
if (modelType === 'diffusion') {
|
|
2118
|
-
return ['image'];
|
|
2119
|
-
}
|
|
2120
|
-
const names = Object.keys(tensorLocations ?? {}).map((name) => name.toLowerCase());
|
|
2121
|
-
const modalities = new Set();
|
|
2122
|
-
if (modelType === 'embedding') {
|
|
2123
|
-
modalities.add('embedding');
|
|
2124
|
-
} else {
|
|
2125
|
-
modalities.add('text');
|
|
2126
|
-
}
|
|
2127
|
-
if (converterConfig?.output?.textOnly === true) {
|
|
2128
|
-
return [...modalities].sort();
|
|
2129
|
-
}
|
|
2130
|
-
if (names.some((name) => name.includes('vision') || name.includes('visual') || name.includes('image'))) {
|
|
2131
|
-
modalities.add('vision');
|
|
2132
|
-
}
|
|
2133
|
-
if (names.some((name) => name.includes('audio'))) {
|
|
2134
|
-
modalities.add('audio');
|
|
2135
|
-
}
|
|
2136
|
-
if (names.some((name) => name.includes('projector') || name.includes('mm_projector'))) {
|
|
2137
|
-
modalities.add('projector');
|
|
2138
|
-
}
|
|
2139
|
-
return [...modalities].sort();
|
|
2140
|
-
}
|
|
2141
|
-
|
|
2142
|
-
function resolveMaterializationProfile(quantizationInfo, inference) {
|
|
2143
|
-
const materialization = typeof inference?.session?.perLayerInputs?.materialization === 'string'
|
|
2144
|
-
? inference.session.perLayerInputs.materialization
|
|
2145
|
-
: 'standard';
|
|
2146
|
-
const perLayerEmbeddings = typeof quantizationInfo?.perLayerEmbeddings === 'string'
|
|
2147
|
-
? quantizationInfo.perLayerEmbeddings
|
|
2148
|
-
: null;
|
|
2149
|
-
return perLayerEmbeddings ? `${materialization}-${perLayerEmbeddings}` : materialization;
|
|
2150
|
-
}
|
|
2151
|
-
|
|
2152
|
-
async function buildArtifactIdentity(options) {
|
|
2153
|
-
const explicit = isPlainRecord(options?.explicitArtifactIdentity)
|
|
2154
|
-
? options.explicitArtifactIdentity
|
|
2155
|
-
: {};
|
|
2156
|
-
const hashString = options?.hashString;
|
|
2157
|
-
if (typeof hashString !== 'function' && Object.keys(explicit).length > 0) {
|
|
2158
|
-
return stripUndefined(explicit);
|
|
2159
|
-
}
|
|
2160
|
-
if (typeof hashString !== 'function') {
|
|
2161
|
-
return null;
|
|
2162
|
-
}
|
|
2163
|
-
|
|
2164
|
-
const sourceIdentity = resolveSourceCheckpointIdentity(explicit, options);
|
|
2165
|
-
const sourceFormat = typeof explicit.sourceFormat === 'string' && explicit.sourceFormat.trim()
|
|
2166
|
-
? explicit.sourceFormat.trim()
|
|
2167
|
-
: resolveArtifactSourceFormat(options);
|
|
2168
|
-
const conversionConfigDigest = explicit.conversionConfigDigest
|
|
2169
|
-
?? (options.conversionConfig
|
|
2170
|
-
? await hashArtifactValue(hashString, options.conversionConfig, 'conversionConfig')
|
|
2171
|
-
: null);
|
|
2172
|
-
const shardSetHash = explicit.shardSetHash ?? explicit.weightPackHash
|
|
2173
|
-
?? await hashArtifactValue(
|
|
2174
|
-
hashString,
|
|
2175
|
-
{
|
|
2176
|
-
hashAlgorithm: options.hashAlgorithm,
|
|
2177
|
-
shards: (options.shards ?? []).map((shard) => ({
|
|
2178
|
-
index: shard.index,
|
|
2179
|
-
filename: shard.filename,
|
|
2180
|
-
size: shard.size,
|
|
2181
|
-
hash: shard.hash,
|
|
2182
|
-
offset: shard.offset,
|
|
2183
|
-
})),
|
|
2184
|
-
},
|
|
2185
|
-
'shardSet'
|
|
2186
|
-
);
|
|
2187
|
-
const modalitySet = Array.isArray(explicit.modalitySet) && explicit.modalitySet.length > 0
|
|
2188
|
-
? [...explicit.modalitySet]
|
|
2189
|
-
: inferArtifactModalitySet(options.modelType, options.tensorLocations, options.converterConfig);
|
|
2190
|
-
const materializationProfile = explicit.materializationProfile
|
|
2191
|
-
?? resolveMaterializationProfile(options.quantizationInfo, options.inference);
|
|
2192
|
-
const weightPackInput = {
|
|
2193
|
-
sourceCheckpointId: sourceIdentity.sourceCheckpointId,
|
|
2194
|
-
sourceFormat,
|
|
2195
|
-
modelType: options.modelType,
|
|
2196
|
-
modalitySet,
|
|
2197
|
-
quantizationInfo: options.quantizationInfo,
|
|
2198
|
-
materializationProfile,
|
|
2199
|
-
shardSetHash,
|
|
2200
|
-
sharding: {
|
|
2201
|
-
shardSizeBytes: options.converterConfig?.sharding?.shardSizeBytes ?? null,
|
|
2202
|
-
},
|
|
2203
|
-
output: {
|
|
2204
|
-
textOnly: options.converterConfig?.output?.textOnly === true,
|
|
2205
|
-
},
|
|
2206
|
-
};
|
|
2207
|
-
const weightPackHash = explicit.weightPackHash
|
|
2208
|
-
?? await hashArtifactValue(hashString, weightPackInput, 'weightPack');
|
|
2209
|
-
const weightPackId = explicit.weightPackId
|
|
2210
|
-
?? `${sanitizeModelId(options.modelId) ?? 'model'}-wp-${digestSuffix(weightPackHash)}`;
|
|
2211
|
-
const manifestVariantHash = await hashArtifactValue(
|
|
2212
|
-
hashString,
|
|
2213
|
-
{
|
|
2214
|
-
weightPackId,
|
|
2215
|
-
modelType: options.modelType,
|
|
2216
|
-
inference: options.inference,
|
|
2217
|
-
config: options.manifestConfig ?? null,
|
|
2218
|
-
},
|
|
2219
|
-
'manifestVariant'
|
|
2220
|
-
);
|
|
2221
|
-
const manifestVariantId = explicit.manifestVariantId
|
|
2222
|
-
?? `${sanitizeModelId(options.modelId) ?? 'model'}-mv-${digestSuffix(manifestVariantHash)}`;
|
|
2223
|
-
|
|
2224
|
-
return stripUndefined({
|
|
2225
|
-
...explicit,
|
|
2226
|
-
sourceCheckpointId: sourceIdentity.sourceCheckpointId,
|
|
2227
|
-
sourceRepo: sourceIdentity.sourceRepo ?? undefined,
|
|
2228
|
-
sourceRevision: sourceIdentity.sourceRevision ?? undefined,
|
|
2229
|
-
sourceFormat,
|
|
2230
|
-
conversionConfigPath: explicit.conversionConfigPath ?? options.conversionConfigPath ?? undefined,
|
|
2231
|
-
conversionConfigDigest: conversionConfigDigest ?? undefined,
|
|
2232
|
-
weightPackId,
|
|
2233
|
-
weightPackHash,
|
|
2234
|
-
shardSetHash,
|
|
2235
|
-
manifestVariantId,
|
|
2236
|
-
modalitySet,
|
|
2237
|
-
materializationProfile,
|
|
2238
|
-
artifactCompleteness: explicit.artifactCompleteness ?? 'complete',
|
|
2239
|
-
});
|
|
2240
|
-
}
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
export function createManifest(
|
|
2244
|
-
modelId,
|
|
2245
|
-
model,
|
|
2246
|
-
shards,
|
|
2247
|
-
tensorLocations,
|
|
2248
|
-
sourceOrOptions
|
|
2249
|
-
) {
|
|
2250
|
-
if (!sourceOrOptions) {
|
|
2251
|
-
throw new Error('Missing manifest options');
|
|
2252
|
-
}
|
|
2253
|
-
const options = typeof sourceOrOptions === 'string' ? { source: sourceOrOptions } : sourceOrOptions ?? {};
|
|
2254
|
-
const source = options.source;
|
|
2255
|
-
if (!source) {
|
|
2256
|
-
throw new Error('Missing manifest source');
|
|
2257
|
-
}
|
|
2258
|
-
const resolvedModelType =
|
|
2259
|
-
options.modelType ??
|
|
2260
|
-
model.modelType ??
|
|
2261
|
-
model.config?.architectures?.[0] ??
|
|
2262
|
-
model.architecture;
|
|
2263
|
-
if (!resolvedModelType) {
|
|
2264
|
-
throw new Error('Missing modelType for manifest');
|
|
2265
|
-
}
|
|
2266
|
-
const isDiffusion = resolvedModelType === 'diffusion';
|
|
2267
|
-
const isEmbedding = resolvedModelType === 'embedding';
|
|
2268
|
-
const architecture = options.architecture ?? model.architecture ?? (
|
|
2269
|
-
isDiffusion ? 'diffusion' : extractArchitecture(model.config, model.ggufConfig)
|
|
2270
|
-
);
|
|
2271
|
-
const rawConfig = model.config || {};
|
|
2272
|
-
const generationConfig = model.generationConfig ?? null;
|
|
2273
|
-
const manifestPolicy = options.manifestConfig ?? null;
|
|
2274
|
-
const resolvedArchitecture = isDiffusion
|
|
2275
|
-
? architecture
|
|
2276
|
-
: resolveIntermediateSizeFromTensors(architecture, model, tensorLocations, rawConfig, modelId);
|
|
2277
|
-
const moeConfig = isDiffusion
|
|
2278
|
-
? null
|
|
2279
|
-
: resolveManifestMoEConfig(model, { ...options, modelId }, rawConfig, resolvedModelType);
|
|
2280
|
-
let inference = options.inference;
|
|
2281
|
-
if (!inference) {
|
|
2282
|
-
throw new Error('inference config is required — use a v1 conversion config');
|
|
2283
|
-
}
|
|
2284
|
-
inference = applyDiffusionGemmaInferenceContract(
|
|
2285
|
-
inference,
|
|
2286
|
-
rawConfig,
|
|
2287
|
-
generationConfig,
|
|
2288
|
-
modelId,
|
|
2289
|
-
resolvedModelType
|
|
2290
|
-
);
|
|
2291
|
-
|
|
2292
|
-
const embeddingOutput = inferEmbeddingOutputConfig(tensorLocations);
|
|
2293
|
-
const hasExplicitEmbeddingPostprocessor = Object.prototype.hasOwnProperty.call(
|
|
2294
|
-
inference?.output ?? {},
|
|
2295
|
-
'embeddingPostprocessor'
|
|
2296
|
-
);
|
|
2297
|
-
const embeddingPostprocessor = hasExplicitEmbeddingPostprocessor
|
|
2298
|
-
? inference?.output?.embeddingPostprocessor
|
|
2299
|
-
: (model.embeddingPostprocessor ?? null);
|
|
2300
|
-
if (embeddingOutput || hasExplicitEmbeddingPostprocessor || embeddingPostprocessor) {
|
|
2301
|
-
inference = {
|
|
2302
|
-
...inference,
|
|
2303
|
-
output: {
|
|
2304
|
-
...embeddingOutput,
|
|
2305
|
-
...inference.output,
|
|
2306
|
-
embeddingPostprocessor,
|
|
2307
|
-
},
|
|
2308
|
-
};
|
|
2309
|
-
}
|
|
2310
|
-
|
|
2311
|
-
const eosTokenId = options.eosTokenId !== undefined
|
|
2312
|
-
? options.eosTokenId
|
|
2313
|
-
: isDiffusion || isEmbedding
|
|
2314
|
-
? null
|
|
2315
|
-
: resolveEosTokenId({
|
|
2316
|
-
config: rawConfig,
|
|
2317
|
-
generationConfig,
|
|
2318
|
-
tokenizer: model.tokenizer ?? model.tokenizerConfig ?? null,
|
|
2319
|
-
tokenizerJson: model.tokenizerJson ?? null,
|
|
2320
|
-
});
|
|
2321
|
-
const resolvedQuantization = options.quantization ?? model.quantization;
|
|
2322
|
-
if (!resolvedQuantization) {
|
|
2323
|
-
throw new Error('Missing quantization for manifest');
|
|
2324
|
-
}
|
|
2325
|
-
const hashAlgorithm = options.hashAlgorithm;
|
|
2326
|
-
if (!hashAlgorithm) {
|
|
2327
|
-
throw new Error('Missing hashAlgorithm for manifest');
|
|
2328
|
-
}
|
|
2329
|
-
|
|
2330
|
-
const isTextOnlyArtifact = options.textOnly === true;
|
|
2331
|
-
const multimodalConfig = isDiffusion || isTextOnlyArtifact
|
|
2332
|
-
? { vision_config: null, audio_config: null }
|
|
2333
|
-
: resolveManifestMultimodalConfig(rawConfig, manifestPolicy);
|
|
2334
|
-
const manifestConfig = isDiffusion
|
|
2335
|
-
? rawConfig
|
|
2336
|
-
: {
|
|
2337
|
-
...(multimodalConfig.vision_config ? { vision_config: multimodalConfig.vision_config } : {}),
|
|
2338
|
-
...(multimodalConfig.audio_config ? { audio_config: multimodalConfig.audio_config } : {}),
|
|
2339
|
-
};
|
|
2340
|
-
|
|
2341
|
-
const manifest = {
|
|
2342
|
-
version: RDRR_VERSION,
|
|
2343
|
-
modelId,
|
|
2344
|
-
modelType: resolvedModelType,
|
|
2345
|
-
quantization: resolvedQuantization,
|
|
2346
|
-
quantizationInfo: options.quantizationInfo,
|
|
2347
|
-
...(options.artifactIdentity ? { artifactIdentity: options.artifactIdentity } : {}),
|
|
2348
|
-
...(options.weightsRef ? { weightsRef: options.weightsRef } : {}),
|
|
2349
|
-
architecture: resolvedArchitecture,
|
|
2350
|
-
moeConfig,
|
|
2351
|
-
inference,
|
|
2352
|
-
shards,
|
|
2353
|
-
tensors: tensorLocations,
|
|
2354
|
-
totalSize: shards.reduce((sum, s) => sum + s.size, 0),
|
|
2355
|
-
hashAlgorithm,
|
|
2356
|
-
eos_token_id: eosTokenId,
|
|
2357
|
-
...(rawConfig.image_token_id !== undefined ? { image_token_id: rawConfig.image_token_id } : {}),
|
|
2358
|
-
...(rawConfig.audio_token_id !== undefined ? { audio_token_id: rawConfig.audio_token_id } : {}),
|
|
2359
|
-
...(rawConfig.video_token_id !== undefined ? { video_token_id: rawConfig.video_token_id } : {}),
|
|
2360
|
-
config: Object.keys(manifestConfig).length > 0 ? manifestConfig : undefined,
|
|
2361
|
-
conversion: options.conversionInfo,
|
|
2362
|
-
metadata: {
|
|
2363
|
-
source,
|
|
2364
|
-
convertedAt: resolveConvertedAt(
|
|
2365
|
-
options.convertedAt
|
|
2366
|
-
?? options.conversionInfo?.convertedAt
|
|
2367
|
-
),
|
|
2368
|
-
},
|
|
2369
|
-
};
|
|
2370
|
-
|
|
2371
|
-
// Include tokenizer if available
|
|
2372
|
-
if (model.tokenizerJson) {
|
|
2373
|
-
manifest.tokenizer = buildBundledTokenizer(
|
|
2374
|
-
model.tokenizerJson,
|
|
2375
|
-
model.tokenizerConfig ?? null,
|
|
2376
|
-
rawConfig
|
|
2377
|
-
);
|
|
2378
|
-
manifest.metadata.hasTokenizer = true;
|
|
2379
|
-
} else {
|
|
2380
|
-
const tokenizer = buildSentencepieceTokenizer(
|
|
2381
|
-
model.tokenizerConfig ?? null,
|
|
2382
|
-
rawConfig,
|
|
2383
|
-
architecture,
|
|
2384
|
-
model.tokenizerModel ?? null
|
|
2385
|
-
);
|
|
2386
|
-
if (tokenizer) {
|
|
2387
|
-
manifest.tokenizer = tokenizer;
|
|
2388
|
-
manifest.metadata.hasTokenizer = true;
|
|
2389
|
-
}
|
|
2390
|
-
}
|
|
2391
|
-
|
|
2392
|
-
return manifest;
|
|
2393
|
-
}
|
|
2394
|
-
|
|
2395
393
|
// ============================================================================
|
|
2396
394
|
// Main Converter (uses I/O adapter)
|
|
2397
395
|
// ============================================================================
|