doppler-gpu 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +210 -0
- package/README.md +63 -38
- package/assets/doppler.svg +57 -1601
- package/package.json +111 -32
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +486 -0
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +885 -744
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +469 -43
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +66 -10
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +69 -4
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +34 -50
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +35 -2
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +148 -3
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +8 -60
- package/src/client/wrap-pipeline-handle.d.ts +82 -0
- package/src/client/wrap-pipeline-handle.js +259 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +28 -157
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +163 -126
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +3593 -1146
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +3 -1
- package/src/config/merge-helpers.js +2 -1
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +65 -7
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +20 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +16 -0
- package/src/config/schema/converter.schema.js +11 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +13 -2
- package/src/config/schema/doppler.schema.js +140 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +39 -1
- package/src/config/schema/execution-v1.schema.js +35 -20
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +14 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +175 -2
- package/src/config/schema/manifest.schema.js +58 -3
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -9
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +183 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
- package/src/config/transforms/execution-graph-transforms.js +829 -168
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +31 -14
- package/src/converter/core.d.ts +65 -0
- package/src/converter/core.js +1153 -22
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +218 -5
- package/src/converter/quantizer.d.ts +24 -4
- package/src/converter/quantizer.js +92 -2
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +296 -38
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1733 -58
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +5 -14
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +192 -12
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +111 -2
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +10 -0
- package/src/formats/rdrr/index.js +10 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +26 -6
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +193 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +111 -2
- package/src/formats/rdrr/validation.js +260 -1
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +45 -23
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +130 -14
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +1 -8
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +458 -16
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +13 -120
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +84 -8
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +25 -22
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +77 -5
- package/src/gpu/kernels/index.js +55 -4
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +44 -2
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +172 -10
- package/src/gpu/kernels/matmul.d.ts +1 -29
- package/src/gpu/kernels/matmul.js +240 -38
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +1 -56
- package/src/gpu/kernels/moe.js +46 -33
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -12
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +6 -4
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +30 -7
- package/src/gpu/kernels/rmsnorm.js +49 -8
- package/src/gpu/kernels/rmsnorm.wgsl +15 -12
- package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +28 -21
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +8 -223
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +43 -0
- package/src/gpu/kernels/utils.js +81 -11
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +10 -12
- package/src/gpu/uniform-cache.js +25 -30
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
- package/src/inference/browser-harness-suite-helpers.js +525 -3
- package/src/inference/browser-harness-text-helpers.d.ts +89 -0
- package/src/inference/browser-harness-text-helpers.js +459 -25
- package/src/inference/browser-harness.d.ts +7 -28
- package/src/inference/browser-harness.js +556 -79
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +31 -0
- package/src/inference/kv-cache/base.js +125 -2
- package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
- package/src/inference/kv-cache/mixed-geometry.js +121 -4
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
- package/src/inference/pipelines/text/attention/projections.js +398 -71
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +542 -56
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +343 -50
- package/src/inference/pipelines/text/attention/types.d.ts +7 -1
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +25 -26
- package/src/inference/pipelines/text/config.js +304 -25
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +25 -29
- package/src/inference/pipelines/text/embed.js +16 -45
- package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
- package/src/inference/pipelines/text/execution-plan.js +21 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
- package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
- package/src/inference/pipelines/text/execution-v1.js +461 -158
- package/src/inference/pipelines/text/ffn/dense.js +214 -6
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +84 -17
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +9 -7
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
- package/src/inference/pipelines/text/generator-steps.js +275 -79
- package/src/inference/pipelines/text/generator.d.ts +38 -5
- package/src/inference/pipelines/text/generator.js +1259 -496
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +13 -3
- package/src/inference/pipelines/text/init.js +213 -725
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +373 -558
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +419 -66
- package/src/inference/pipelines/text/logits/index.d.ts +3 -3
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +42 -4
- package/src/inference/pipelines/text/ops.js +20 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +119 -31
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +153 -9
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
- package/src/inference/pipelines/text/types.d.ts +92 -81
- package/src/inference/pipelines/text/weights.d.ts +4 -52
- package/src/inference/pipelines/text/weights.js +6 -56
- package/src/inference/pipelines/text.d.ts +33 -1
- package/src/inference/pipelines/text.js +375 -55
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +341 -174
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +14 -0
- package/src/loader/doppler-loader.js +378 -25
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +276 -11
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +61 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +52 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +36 -8
- package/src/loader/tensors/tensor-loader.js +521 -22
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +13 -42
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +57 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +162 -10
- package/src/rules/inference/capability-transforms.rules.json +111 -0
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +117 -16
- package/src/rules/inference/ffn.rules.json +24 -2
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +17 -0
- package/src/rules/rule-registry.js +59 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +241 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +31 -4
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +53 -9
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +8 -3
- package/src/tooling/command-api.d.ts +22 -1
- package/src/tooling/command-api.js +11 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +14 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +3 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +571 -37
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +107 -17
- package/src/tooling/node-command-runner.js +43 -2
- package/src/tooling/node-convert-worker-pool.js +15 -0
- package/src/tooling/node-convert-worker.js +11 -1
- package/src/tooling/node-converter.js +102 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +112 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +10 -5
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +110 -8
- package/src/tooling/source-runtime-bundle.d.ts +3 -0
- package/src/tooling/source-runtime-bundle.js +30 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +25 -12
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -229,7 +229,7 @@ function formatGemma4(messages, options) {
|
|
|
229
229
|
if (options?.thinking === true) {
|
|
230
230
|
parts.push('<|turn>model\n<|think|>\n');
|
|
231
231
|
} else {
|
|
232
|
-
parts.push('<|turn>model\n');
|
|
232
|
+
parts.push('<|turn>model\n<|channel>thought\n<channel|>');
|
|
233
233
|
}
|
|
234
234
|
return parts.join('');
|
|
235
235
|
}
|
|
@@ -440,10 +440,3 @@ export function formatChatMessages(messages, templateType, options) {
|
|
|
440
440
|
}
|
|
441
441
|
throw new Error(`Unrecognized chat template type: ${templateType}`);
|
|
442
442
|
}
|
|
443
|
-
|
|
444
|
-
export const formatGemmaChat = formatTurnBased;
|
|
445
|
-
export const formatGemma4Chat = formatGemma4;
|
|
446
|
-
export const formatLlama3Chat = formatHeaderBased;
|
|
447
|
-
export const formatGptOssChat = formatChannelBased;
|
|
448
|
-
export const formatQwenChat = formatQwen;
|
|
449
|
-
export const formatTranslateGemmaChat = formatTranslateGemma;
|
|
@@ -46,6 +46,8 @@ export interface RawConfig {
|
|
|
46
46
|
n_head?: number;
|
|
47
47
|
attentionHeadCount?: number;
|
|
48
48
|
num_key_value_heads?: number;
|
|
49
|
+
num_global_key_value_heads?: number;
|
|
50
|
+
num_global_kv_heads?: number;
|
|
49
51
|
attentionHeadCountKV?: number;
|
|
50
52
|
head_dim?: number;
|
|
51
53
|
global_head_dim?: number;
|
|
@@ -107,15 +109,23 @@ export interface TensorInfo {
|
|
|
107
109
|
dtype?: string;
|
|
108
110
|
}
|
|
109
111
|
|
|
110
|
-
export
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
112
|
+
export interface ParsedSessionSettings {
|
|
113
|
+
prefillChunkSubmitMode: 'sync' | 'async' | null;
|
|
114
|
+
prefillTokenChunkSize: number | null;
|
|
115
|
+
useFlashPrefillAttention: boolean | null;
|
|
116
|
+
useWideTileQ4KPrefill: boolean | null;
|
|
117
|
+
useWideTileQ4KDecode: boolean | null;
|
|
118
|
+
useSandwichRMSNormPairFusion: boolean | null;
|
|
119
|
+
usePostFfnNextInputRMSNormPairFusion: boolean | null;
|
|
120
|
+
useFusedQKVSplitQKNorm: boolean | null;
|
|
121
|
+
useFusedQKVSplitQKNormRoPE: boolean | null;
|
|
122
|
+
retainQ4KMaterialization: boolean | null;
|
|
116
123
|
}
|
|
117
124
|
|
|
125
|
+
export type Manifest = RuntimeModelContract;
|
|
126
|
+
|
|
118
127
|
export interface ParsedModelConfig {
|
|
128
|
+
modelType: string;
|
|
119
129
|
numLayers: number;
|
|
120
130
|
hiddenSize: number;
|
|
121
131
|
intermediateSize: number;
|
|
@@ -123,6 +133,7 @@ export interface ParsedModelConfig {
|
|
|
123
133
|
maxIntermediateSize: number;
|
|
124
134
|
numHeads: number;
|
|
125
135
|
numKVHeads: number;
|
|
136
|
+
numGlobalKVHeads: number | null;
|
|
126
137
|
headDim: number;
|
|
127
138
|
globalHeadDim: number | null;
|
|
128
139
|
vocabSize: number;
|
|
@@ -133,7 +144,8 @@ export interface ParsedModelConfig {
|
|
|
133
144
|
useMoE: boolean;
|
|
134
145
|
numExperts: number;
|
|
135
146
|
moeTopK: number;
|
|
136
|
-
expertFormat: 'mixtral' | 'gpt-oss' | null;
|
|
147
|
+
expertFormat: 'mixtral' | 'gpt-oss' | 'gemma4' | null;
|
|
148
|
+
moeExpertIntermediateSize: number;
|
|
137
149
|
slidingWindow: number | null;
|
|
138
150
|
ropeTheta: number;
|
|
139
151
|
ropeLocalTheta: number | null;
|
|
@@ -165,6 +177,7 @@ export interface ParsedModelConfig {
|
|
|
165
177
|
embeddingVocabSize: number | null;
|
|
166
178
|
embeddingPostprocessor: ManifestEmbeddingPostprocessorSchema | null;
|
|
167
179
|
hiddenActivation: ActivationType;
|
|
180
|
+
ffnBranchMode: 'auto' | 'dense' | 'moe' | 'dense_plus_moe';
|
|
168
181
|
useDoubleWideMlp: boolean;
|
|
169
182
|
swigluLimit: number | null;
|
|
170
183
|
stopTokenIds: number[];
|
|
@@ -181,6 +194,8 @@ export interface ParsedModelConfig {
|
|
|
181
194
|
finalLogitSoftcapping: number | null;
|
|
182
195
|
attnLogitSoftcapping: number | null;
|
|
183
196
|
queryKeyNorm: boolean;
|
|
197
|
+
queryKeyNormLayers: number[] | null;
|
|
198
|
+
queryKeyNormWeightLayers: number[] | null;
|
|
184
199
|
valueNorm: boolean;
|
|
185
200
|
attentionOutputGate: boolean;
|
|
186
201
|
queryPreAttnScalar: number;
|
|
@@ -189,7 +204,9 @@ export interface ParsedModelConfig {
|
|
|
189
204
|
chatTemplateEnabled: boolean;
|
|
190
205
|
chatTemplateThinking: boolean | null;
|
|
191
206
|
decodeStrategy: 'incremental' | 'replay_prefill';
|
|
207
|
+
diffusionGemma: ManifestInferenceSchema['diffusionGemma'];
|
|
192
208
|
perLayerInputsSession: ExecutionV1PerLayerInputsSessionSchema;
|
|
209
|
+
sessionSettings: ParsedSessionSettings;
|
|
193
210
|
kernelPath?: KernelPathRef;
|
|
194
211
|
visionConfig?: VisionConfig | null;
|
|
195
212
|
audioConfig?: AudioEncoderConfig | null;
|
|
@@ -249,12 +266,7 @@ export interface AudioEncoderConfig {
|
|
|
249
266
|
|
|
250
267
|
export function getStopTokenIds(manifest: Manifest): number[];
|
|
251
268
|
export function resolveLayerIntermediateSize(config: ParsedModelConfig, layerIdx: number): number;
|
|
252
|
-
|
|
253
|
-
/**
|
|
254
|
-
* Validate that a chatTemplate.type is a known formatter type.
|
|
255
|
-
* Logs a warning for unknown types. Returns true if valid or null.
|
|
256
|
-
*/
|
|
257
|
-
export function validateChatTemplateType(type: string | null | undefined, modelId: string): boolean;
|
|
269
|
+
export function assertSupportedManifestInference(manifest: Manifest): void;
|
|
258
270
|
|
|
259
271
|
/**
|
|
260
272
|
* Extended manifest with inference config for manifest-first parsing.
|
|
@@ -270,24 +282,11 @@ export interface ManifestWithInference {
|
|
|
270
282
|
eos_token_id: number | number[];
|
|
271
283
|
}
|
|
272
284
|
|
|
273
|
-
/**
|
|
274
|
-
* Check if manifest has inference config for manifest-first parsing.
|
|
275
|
-
*/
|
|
276
|
-
export function hasManifestInference(manifest: Manifest): manifest is Manifest & { inference: ManifestInferenceSchema };
|
|
277
|
-
|
|
278
285
|
export function validateRequiredInferenceFields(
|
|
279
286
|
inf: ManifestInferenceSchema,
|
|
280
287
|
modelId: string
|
|
281
288
|
): void;
|
|
282
289
|
|
|
283
|
-
/**
|
|
284
|
-
* Convert MergedConfig to ParsedModelConfig.
|
|
285
|
-
*/
|
|
286
|
-
export function toParsedConfigFromMerged(
|
|
287
|
-
merged: MergedConfig,
|
|
288
|
-
manifest: ManifestWithInference
|
|
289
|
-
): ParsedModelConfig;
|
|
290
|
-
|
|
291
290
|
/**
|
|
292
291
|
* Parse model config from manifest using manifest-first resolution.
|
|
293
292
|
*/
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
PER_LAYER_INPUT_PREFETCH_MODES,
|
|
8
8
|
PER_LAYER_INPUT_GPU_UPLOAD_MODES,
|
|
9
9
|
PER_LAYER_INPUT_HOT_CACHE_MODES,
|
|
10
|
+
PREFILL_CHUNK_SUBMIT_MODES,
|
|
10
11
|
} from '../../../config/schema/execution-v1.schema.js';
|
|
11
12
|
|
|
12
13
|
const UNSUPPORTED_RUNTIME_MODEL_TYPES = new Set(['mamba', 'rwkv']);
|
|
@@ -34,7 +35,7 @@ const KNOWN_CHAT_TEMPLATE_TYPES = new Set([
|
|
|
34
35
|
* @param {string} modelId - Model identifier for diagnostic messages.
|
|
35
36
|
* @returns {boolean} True if the type is valid or null.
|
|
36
37
|
*/
|
|
37
|
-
|
|
38
|
+
function validateChatTemplateType(type, modelId) {
|
|
38
39
|
if (type === null || type === undefined) return true;
|
|
39
40
|
if (KNOWN_CHAT_TEMPLATE_TYPES.has(type)) return true;
|
|
40
41
|
throw new Error(
|
|
@@ -196,6 +197,95 @@ function resolvePerLayerInputsSession(inferenceConfig, modelId) {
|
|
|
196
197
|
};
|
|
197
198
|
}
|
|
198
199
|
|
|
200
|
+
function resolveSessionSettings(inferenceConfig, modelId) {
|
|
201
|
+
// All four fields are optional on the manifest today for backwards compatibility;
|
|
202
|
+
// when absent we fall through to the runtime config (getRuntimeConfig()). When
|
|
203
|
+
// present on the manifest, manifest wins unless an explicit runtime profile overrides.
|
|
204
|
+
// The merge layer in src/config/merge.js already applies that precedence — this
|
|
205
|
+
// resolver just validates the manifest-supplied values and normalizes them.
|
|
206
|
+
const session = inferenceConfig?.session;
|
|
207
|
+
const submit = session?.prefillChunkSubmitMode;
|
|
208
|
+
if (submit !== undefined && submit !== null && !PREFILL_CHUNK_SUBMIT_MODES.includes(submit)) {
|
|
209
|
+
throw new Error(
|
|
210
|
+
`Manifest "${modelId}" has invalid inference.session.prefillChunkSubmitMode ` +
|
|
211
|
+
`"${String(submit)}"; expected one of ${PREFILL_CHUNK_SUBMIT_MODES.join(', ')}.`
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
const tokenChunk = session?.prefillTokenChunkSize;
|
|
215
|
+
if (
|
|
216
|
+
tokenChunk !== undefined
|
|
217
|
+
&& tokenChunk !== null
|
|
218
|
+
&& (!Number.isInteger(tokenChunk) || tokenChunk <= 0)
|
|
219
|
+
) {
|
|
220
|
+
throw new Error(
|
|
221
|
+
`Manifest "${modelId}" has invalid inference.session.prefillTokenChunkSize ` +
|
|
222
|
+
`"${String(tokenChunk)}"; expected null or a positive integer.`
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
const flash = session?.useFlashPrefillAttention;
|
|
226
|
+
if (flash !== undefined && flash !== null && typeof flash !== 'boolean') {
|
|
227
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFlashPrefillAttention "${String(flash)}"; expected boolean.`);
|
|
228
|
+
}
|
|
229
|
+
const wide = session?.useWideTileQ4KPrefill;
|
|
230
|
+
if (wide !== undefined && wide !== null && typeof wide !== 'boolean') {
|
|
231
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileQ4KPrefill "${String(wide)}"; expected boolean.`);
|
|
232
|
+
}
|
|
233
|
+
const wideDecode = session?.useWideTileQ4KDecode;
|
|
234
|
+
if (wideDecode !== undefined && wideDecode !== null && typeof wideDecode !== 'boolean') {
|
|
235
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileQ4KDecode "${String(wideDecode)}"; expected boolean.`);
|
|
236
|
+
}
|
|
237
|
+
const sandwichRmsNormPair = session?.useSandwichRMSNormPairFusion;
|
|
238
|
+
if (sandwichRmsNormPair !== undefined && sandwichRmsNormPair !== null && typeof sandwichRmsNormPair !== 'boolean') {
|
|
239
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useSandwichRMSNormPairFusion "${String(sandwichRmsNormPair)}"; expected boolean.`);
|
|
240
|
+
}
|
|
241
|
+
const postFfnNextInputRmsNormPair = session?.usePostFfnNextInputRMSNormPairFusion;
|
|
242
|
+
if (postFfnNextInputRmsNormPair !== undefined && postFfnNextInputRmsNormPair !== null && typeof postFfnNextInputRmsNormPair !== 'boolean') {
|
|
243
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.usePostFfnNextInputRMSNormPairFusion "${String(postFfnNextInputRmsNormPair)}"; expected boolean.`);
|
|
244
|
+
}
|
|
245
|
+
const fusedQKVSplitQKNorm = session?.useFusedQKVSplitQKNorm;
|
|
246
|
+
if (fusedQKVSplitQKNorm !== undefined && fusedQKVSplitQKNorm !== null && typeof fusedQKVSplitQKNorm !== 'boolean') {
|
|
247
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFusedQKVSplitQKNorm "${String(fusedQKVSplitQKNorm)}"; expected boolean.`);
|
|
248
|
+
}
|
|
249
|
+
const fusedQKVSplitQKNormRoPE = session?.useFusedQKVSplitQKNormRoPE;
|
|
250
|
+
if (fusedQKVSplitQKNormRoPE !== undefined && fusedQKVSplitQKNormRoPE !== null && typeof fusedQKVSplitQKNormRoPE !== 'boolean') {
|
|
251
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFusedQKVSplitQKNormRoPE "${String(fusedQKVSplitQKNormRoPE)}"; expected boolean.`);
|
|
252
|
+
}
|
|
253
|
+
const retain = session?.retainQ4KMaterialization;
|
|
254
|
+
if (retain !== undefined && retain !== null && typeof retain !== 'boolean') {
|
|
255
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.retainQ4KMaterialization "${String(retain)}"; expected boolean.`);
|
|
256
|
+
}
|
|
257
|
+
return {
|
|
258
|
+
prefillChunkSubmitMode: submit ?? null,
|
|
259
|
+
prefillTokenChunkSize: tokenChunk ?? null,
|
|
260
|
+
useFlashPrefillAttention: flash ?? null,
|
|
261
|
+
useWideTileQ4KPrefill: wide ?? null,
|
|
262
|
+
useWideTileQ4KDecode: wideDecode ?? null,
|
|
263
|
+
useSandwichRMSNormPairFusion: sandwichRmsNormPair ?? null,
|
|
264
|
+
usePostFfnNextInputRMSNormPairFusion: postFfnNextInputRmsNormPair ?? null,
|
|
265
|
+
useFusedQKVSplitQKNorm: fusedQKVSplitQKNorm ?? null,
|
|
266
|
+
useFusedQKVSplitQKNormRoPE: fusedQKVSplitQKNormRoPE ?? null,
|
|
267
|
+
retainQ4KMaterialization: retain ?? null,
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
function resolveLargeWeightsConfig(inferenceConfig, modelId) {
|
|
272
|
+
const lw = inferenceConfig?.largeWeights;
|
|
273
|
+
if (lw === undefined || lw === null) return { gpuResidentOverrides: null };
|
|
274
|
+
if (typeof lw !== 'object' || Array.isArray(lw)) {
|
|
275
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.largeWeights (must be object).`);
|
|
276
|
+
}
|
|
277
|
+
const overrides = lw.gpuResidentOverrides;
|
|
278
|
+
if (overrides !== null && overrides !== undefined) {
|
|
279
|
+
if (!Array.isArray(overrides) || !overrides.every((v) => typeof v === 'string' && v.length > 0)) {
|
|
280
|
+
throw new Error(
|
|
281
|
+
`Manifest "${modelId}" has invalid inference.largeWeights.gpuResidentOverrides; ` +
|
|
282
|
+
`expected array of non-empty strings or null.`
|
|
283
|
+
);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
return { gpuResidentOverrides: overrides ?? null };
|
|
287
|
+
}
|
|
288
|
+
|
|
199
289
|
// =============================================================================
|
|
200
290
|
// Model Detection Functions
|
|
201
291
|
// =============================================================================
|
|
@@ -541,10 +631,37 @@ export function resolveLayerIntermediateSize(config, layerIdx) {
|
|
|
541
631
|
// =============================================================================
|
|
542
632
|
|
|
543
633
|
|
|
544
|
-
|
|
634
|
+
function hasManifestInference(manifest) {
|
|
545
635
|
return 'inference' in manifest && manifest.inference != null;
|
|
546
636
|
}
|
|
547
637
|
|
|
638
|
+
function normalizeUnsupportedText(value) {
|
|
639
|
+
if (typeof value !== 'string') return '';
|
|
640
|
+
return value.trim();
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
export function assertSupportedManifestInference(manifest) {
|
|
644
|
+
const modelId = manifest?.modelId ?? 'unknown';
|
|
645
|
+
const unsupported = manifest?.inference?.unsupported;
|
|
646
|
+
if (unsupported == null) {
|
|
647
|
+
return;
|
|
648
|
+
}
|
|
649
|
+
if (typeof unsupported !== 'object' || Array.isArray(unsupported)) {
|
|
650
|
+
throw new Error(
|
|
651
|
+
`Manifest "${modelId}" has invalid inference.unsupported. ` +
|
|
652
|
+
'Expected null or an object with code, message, and recommendation fields.'
|
|
653
|
+
);
|
|
654
|
+
}
|
|
655
|
+
const code = normalizeUnsupportedText(unsupported.code) || 'unsupported-manifest-contract';
|
|
656
|
+
const message = normalizeUnsupportedText(unsupported.message);
|
|
657
|
+
const recommendation = normalizeUnsupportedText(unsupported.recommendation);
|
|
658
|
+
throw new Error(
|
|
659
|
+
`Manifest "${modelId}" is not supported by Doppler runtime (${code}).` +
|
|
660
|
+
(message ? ` ${message}` : '') +
|
|
661
|
+
(recommendation ? ` ${recommendation}` : '')
|
|
662
|
+
);
|
|
663
|
+
}
|
|
664
|
+
|
|
548
665
|
|
|
549
666
|
export function validateRequiredInferenceFields(inf, modelId) {
|
|
550
667
|
inf = inf ?? {};
|
|
@@ -564,6 +681,36 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
564
681
|
if (inf.attention.queryKeyNorm == null) {
|
|
565
682
|
errors.push('attention.queryKeyNorm is required');
|
|
566
683
|
}
|
|
684
|
+
if (
|
|
685
|
+
inf.attention.queryKeyNormLayers !== undefined
|
|
686
|
+
&& inf.attention.queryKeyNormLayers !== null
|
|
687
|
+
) {
|
|
688
|
+
if (!Array.isArray(inf.attention.queryKeyNormLayers)) {
|
|
689
|
+
errors.push('attention.queryKeyNormLayers must be null or an array of layer indices');
|
|
690
|
+
} else {
|
|
691
|
+
for (const layerIdx of inf.attention.queryKeyNormLayers) {
|
|
692
|
+
if (!Number.isInteger(layerIdx) || layerIdx < 0) {
|
|
693
|
+
errors.push('attention.queryKeyNormLayers must contain only non-negative integer layer indices');
|
|
694
|
+
break;
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
if (
|
|
700
|
+
inf.attention.queryKeyNormWeightLayers !== undefined
|
|
701
|
+
&& inf.attention.queryKeyNormWeightLayers !== null
|
|
702
|
+
) {
|
|
703
|
+
if (!Array.isArray(inf.attention.queryKeyNormWeightLayers)) {
|
|
704
|
+
errors.push('attention.queryKeyNormWeightLayers must be null or an array of layer indices');
|
|
705
|
+
} else {
|
|
706
|
+
for (const layerIdx of inf.attention.queryKeyNormWeightLayers) {
|
|
707
|
+
if (!Number.isInteger(layerIdx) || layerIdx < 0) {
|
|
708
|
+
errors.push('attention.queryKeyNormWeightLayers must contain only non-negative integer layer indices');
|
|
709
|
+
break;
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
}
|
|
567
714
|
if (inf.attention.valueNorm == null) {
|
|
568
715
|
errors.push('attention.valueNorm is required');
|
|
569
716
|
}
|
|
@@ -605,6 +752,15 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
605
752
|
if (inf.ffn.gatedActivation == null) {
|
|
606
753
|
errors.push('ffn.gatedActivation is required');
|
|
607
754
|
}
|
|
755
|
+
if (inf.ffn.branchMode !== undefined) {
|
|
756
|
+
const normalizedBranchMode = typeof inf.ffn.branchMode === 'string'
|
|
757
|
+
? inf.ffn.branchMode.trim().toLowerCase()
|
|
758
|
+
: '';
|
|
759
|
+
const supportedBranchModes = new Set(['auto', 'dense', 'moe', 'dense_plus_moe']);
|
|
760
|
+
if (!supportedBranchModes.has(normalizedBranchMode)) {
|
|
761
|
+
errors.push('ffn.branchMode must be one of: auto, dense, moe, dense_plus_moe');
|
|
762
|
+
}
|
|
763
|
+
}
|
|
608
764
|
if (inf.ffn.useDoubleWideMlp == null) {
|
|
609
765
|
errors.push('ffn.useDoubleWideMlp is required');
|
|
610
766
|
}
|
|
@@ -868,6 +1024,17 @@ function resolveVisionConfig(rawConfig, manifest) {
|
|
|
868
1024
|
}
|
|
869
1025
|
return Math.trunc(number);
|
|
870
1026
|
};
|
|
1027
|
+
const resolveRequiredNonNegativeInteger = (keys, label) => {
|
|
1028
|
+
const value = resolveRequiredVisionField(keys, label);
|
|
1029
|
+
const number = Number(value);
|
|
1030
|
+
if (!Number.isFinite(number) || number < 0 || Math.floor(number) !== number) {
|
|
1031
|
+
throw new Error(
|
|
1032
|
+
`Manifest "${modelId}" has invalid vision_config.${label}=${JSON.stringify(value)}. ` +
|
|
1033
|
+
'Expected a non-negative integer.'
|
|
1034
|
+
);
|
|
1035
|
+
}
|
|
1036
|
+
return Math.trunc(number);
|
|
1037
|
+
};
|
|
871
1038
|
const resolveRequiredPositiveNumber = (value, label) => {
|
|
872
1039
|
const number = Number(value);
|
|
873
1040
|
if (!Number.isFinite(number) || number <= 0) {
|
|
@@ -887,14 +1054,21 @@ function resolveVisionConfig(rawConfig, manifest) {
|
|
|
887
1054
|
}
|
|
888
1055
|
|
|
889
1056
|
if (visionArchitecture === 'gemma4') {
|
|
1057
|
+
const depth = resolveRequiredNonNegativeInteger(['depth', 'num_hidden_layers'], 'num_hidden_layers');
|
|
1058
|
+
const isEncoderFree = (depth === 0);
|
|
1059
|
+
|
|
890
1060
|
const hiddenSize = resolveRequiredPositiveInteger(['hidden_size'], 'hidden_size');
|
|
891
|
-
const numHeads =
|
|
1061
|
+
const numHeads = isEncoderFree
|
|
1062
|
+
? (vc.num_heads !== undefined || vc.num_attention_heads !== undefined ? resolveRequiredPositiveInteger(['num_heads', 'num_attention_heads'], 'num_attention_heads') : 1)
|
|
1063
|
+
: resolveRequiredPositiveInteger(['num_heads', 'num_attention_heads'], 'num_attention_heads');
|
|
892
1064
|
const ropeParameters = vc.rope_parameters;
|
|
893
1065
|
if (!ropeParameters || typeof ropeParameters !== 'object') {
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
1066
|
+
if (!isEncoderFree) {
|
|
1067
|
+
throw new Error(
|
|
1068
|
+
`Manifest "${modelId}" is missing vision_config.rope_parameters. ` +
|
|
1069
|
+
'Re-convert the model with explicit Gemma 4 vision RoPE metadata.'
|
|
1070
|
+
);
|
|
1071
|
+
}
|
|
898
1072
|
}
|
|
899
1073
|
const hiddenActivation = String(resolveRequiredVisionField(['hidden_activation'], 'hidden_activation')).trim();
|
|
900
1074
|
if (hiddenActivation !== 'gelu' && hiddenActivation !== 'gelu_pytorch_tanh') {
|
|
@@ -915,12 +1089,18 @@ function resolveVisionConfig(rawConfig, manifest) {
|
|
|
915
1089
|
}
|
|
916
1090
|
|
|
917
1091
|
return {
|
|
918
|
-
depth
|
|
1092
|
+
depth,
|
|
919
1093
|
hiddenSize,
|
|
920
|
-
intermediateSize:
|
|
1094
|
+
intermediateSize: isEncoderFree
|
|
1095
|
+
? (vc.intermediate_size !== undefined ? resolveRequiredPositiveInteger(['intermediate_size'], 'intermediate_size') : 1)
|
|
1096
|
+
: resolveRequiredPositiveInteger(['intermediate_size'], 'intermediate_size'),
|
|
921
1097
|
numHeads,
|
|
922
|
-
numKeyValueHeads:
|
|
923
|
-
|
|
1098
|
+
numKeyValueHeads: isEncoderFree
|
|
1099
|
+
? (vc.num_key_value_heads !== undefined ? resolveRequiredPositiveInteger(['num_key_value_heads'], 'num_key_value_heads') : 1)
|
|
1100
|
+
: resolveRequiredPositiveInteger(['num_key_value_heads'], 'num_key_value_heads'),
|
|
1101
|
+
headDim: isEncoderFree
|
|
1102
|
+
? (vc.head_dim !== undefined || vc.global_head_dim !== undefined ? resolveRequiredPositiveInteger(['head_dim', 'global_head_dim'], 'head_dim') : 1)
|
|
1103
|
+
: resolveRequiredPositiveInteger(['head_dim', 'global_head_dim'], 'head_dim'),
|
|
924
1104
|
outHiddenSize: vc.out_hidden_size ?? vc.output_proj_dims ?? null,
|
|
925
1105
|
patchSize: resolveRequiredPositiveInteger(['patch_size'], 'patch_size'),
|
|
926
1106
|
poolingKernelSize: resolveRequiredPositiveInteger(['pooling_kernel_size'], 'pooling_kernel_size'),
|
|
@@ -928,7 +1108,9 @@ function resolveVisionConfig(rawConfig, manifest) {
|
|
|
928
1108
|
temporalPatchSize: vc.temporal_patch_size ?? null,
|
|
929
1109
|
positionEmbeddingSize: resolveRequiredPositiveInteger(['position_embedding_size'], 'position_embedding_size'),
|
|
930
1110
|
defaultOutputLength: resolveRequiredPositiveInteger(['default_output_length'], 'default_output_length'),
|
|
931
|
-
ropeTheta:
|
|
1111
|
+
ropeTheta: isEncoderFree && (!ropeParameters || ropeParameters.rope_theta === undefined)
|
|
1112
|
+
? 10000
|
|
1113
|
+
: resolveRequiredPositiveNumber(ropeParameters.rope_theta, 'vision_config.rope_parameters.rope_theta'),
|
|
932
1114
|
eps: resolveRequiredPositiveNumber(
|
|
933
1115
|
resolveRequiredVisionField(['eps', 'rms_norm_eps'], 'rms_norm_eps'),
|
|
934
1116
|
'vision_config.rms_norm_eps'
|
|
@@ -1026,6 +1208,16 @@ function resolveAudioConfig(rawConfig, manifest) {
|
|
|
1026
1208
|
}
|
|
1027
1209
|
return Math.trunc(number);
|
|
1028
1210
|
};
|
|
1211
|
+
const resolveRequiredNonNegativeInteger = (value, label) => {
|
|
1212
|
+
const number = Number(value);
|
|
1213
|
+
if (!Number.isFinite(number) || number < 0 || Math.floor(number) !== number) {
|
|
1214
|
+
throw new Error(
|
|
1215
|
+
`Manifest "${modelId}" has invalid audio_config.${label}=${JSON.stringify(value)}. ` +
|
|
1216
|
+
'Expected a non-negative integer.'
|
|
1217
|
+
);
|
|
1218
|
+
}
|
|
1219
|
+
return Math.trunc(number);
|
|
1220
|
+
};
|
|
1029
1221
|
const resolveRequiredPositiveNumber = (value, label) => {
|
|
1030
1222
|
const number = Number(value);
|
|
1031
1223
|
if (!Number.isFinite(number) || number <= 0) {
|
|
@@ -1044,31 +1236,50 @@ function resolveAudioConfig(rawConfig, manifest) {
|
|
|
1044
1236
|
);
|
|
1045
1237
|
}
|
|
1046
1238
|
|
|
1239
|
+
const depth = resolveRequiredNonNegativeInteger(ac.num_hidden_layers, 'num_hidden_layers');
|
|
1240
|
+
const isEncoderFree = (depth === 0);
|
|
1241
|
+
|
|
1047
1242
|
const hiddenSize = resolveRequiredPositiveInteger(ac.hidden_size, 'hidden_size');
|
|
1048
|
-
const numAttentionHeads =
|
|
1243
|
+
const numAttentionHeads = isEncoderFree
|
|
1244
|
+
? (ac.num_attention_heads !== undefined ? resolveRequiredPositiveInteger(ac.num_attention_heads, 'num_attention_heads') : 1)
|
|
1245
|
+
: resolveRequiredPositiveInteger(ac.num_attention_heads, 'num_attention_heads');
|
|
1049
1246
|
const headDim = Math.trunc(hiddenSize / numAttentionHeads);
|
|
1050
1247
|
|
|
1051
|
-
if (!
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1248
|
+
if (!isEncoderFree) {
|
|
1249
|
+
if (!Array.isArray(ac.subsampling_conv_channels) || ac.subsampling_conv_channels.length < 1) {
|
|
1250
|
+
throw new Error(
|
|
1251
|
+
`Manifest "${modelId}" is missing audio_config.subsampling_conv_channels array.`
|
|
1252
|
+
);
|
|
1253
|
+
}
|
|
1055
1254
|
}
|
|
1056
1255
|
|
|
1057
1256
|
return {
|
|
1058
1257
|
audioArchitecture,
|
|
1059
|
-
depth
|
|
1258
|
+
depth,
|
|
1060
1259
|
hiddenSize,
|
|
1061
1260
|
numAttentionHeads,
|
|
1062
1261
|
headDim,
|
|
1063
|
-
convKernelSize:
|
|
1064
|
-
|
|
1262
|
+
convKernelSize: isEncoderFree
|
|
1263
|
+
? (ac.conv_kernel_size !== undefined ? resolveRequiredPositiveInteger(ac.conv_kernel_size, 'conv_kernel_size') : 1)
|
|
1264
|
+
: resolveRequiredPositiveInteger(ac.conv_kernel_size, 'conv_kernel_size'),
|
|
1265
|
+
subsamplingConvChannels: isEncoderFree
|
|
1266
|
+
? (ac.subsampling_conv_channels ? ac.subsampling_conv_channels.map(Number) : [])
|
|
1267
|
+
: ac.subsampling_conv_channels.map(Number),
|
|
1065
1268
|
outputProjDims: resolveRequiredPositiveInteger(ac.output_proj_dims, 'output_proj_dims'),
|
|
1066
|
-
attentionContextLeft:
|
|
1269
|
+
attentionContextLeft: isEncoderFree
|
|
1270
|
+
? (ac.attention_context_left !== undefined ? resolveRequiredPositiveInteger(ac.attention_context_left, 'attention_context_left') : 1)
|
|
1271
|
+
: resolveRequiredPositiveInteger(ac.attention_context_left, 'attention_context_left'),
|
|
1067
1272
|
attentionContextRight: Number(ac.attention_context_right ?? 0),
|
|
1068
|
-
attentionChunkSize:
|
|
1069
|
-
|
|
1273
|
+
attentionChunkSize: isEncoderFree
|
|
1274
|
+
? (ac.attention_chunk_size !== undefined ? resolveRequiredPositiveInteger(ac.attention_chunk_size, 'attention_chunk_size') : 1)
|
|
1275
|
+
: resolveRequiredPositiveInteger(ac.attention_chunk_size, 'attention_chunk_size'),
|
|
1276
|
+
attentionLogitCap: isEncoderFree
|
|
1277
|
+
? (ac.attention_logit_cap !== undefined ? resolveRequiredPositiveNumber(ac.attention_logit_cap, 'attention_logit_cap') : 1.0)
|
|
1278
|
+
: resolveRequiredPositiveNumber(ac.attention_logit_cap, 'attention_logit_cap'),
|
|
1070
1279
|
attentionInvalidLogitsValue: Number(ac.attention_invalid_logits_value ?? -1e9),
|
|
1071
|
-
residualWeight:
|
|
1280
|
+
residualWeight: isEncoderFree
|
|
1281
|
+
? (ac.residual_weight !== undefined ? resolveRequiredPositiveNumber(ac.residual_weight, 'residual_weight') : 1.0)
|
|
1282
|
+
: resolveRequiredPositiveNumber(ac.residual_weight, 'residual_weight'),
|
|
1072
1283
|
rmsNormEps: resolveRequiredPositiveNumber(ac.rms_norm_eps ?? 1e-6, 'rms_norm_eps'),
|
|
1073
1284
|
hiddenAct: String(ac.hidden_act ?? 'silu').trim(),
|
|
1074
1285
|
useClippedLinears: ac.use_clipped_linears === true,
|
|
@@ -1120,7 +1331,7 @@ function parseLinearNormMode(value, sharedFlag = null, modelId = 'unknown') {
|
|
|
1120
1331
|
}
|
|
1121
1332
|
|
|
1122
1333
|
|
|
1123
|
-
|
|
1334
|
+
function toParsedConfigFromMerged(merged, manifest) {
|
|
1124
1335
|
const mergedConfig = merged?.config ?? manifest.config ?? {};
|
|
1125
1336
|
const rawConfig = mergedConfig.text_config ?? mergedConfig;
|
|
1126
1337
|
const config = {
|
|
@@ -1298,6 +1509,16 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1298
1509
|
const numExperts = useMoE ? moeConfig.numExperts : 0;
|
|
1299
1510
|
const moeTopK = useMoE ? moeConfig.numExpertsPerToken : 0;
|
|
1300
1511
|
const expertFormat = useMoE ? moeConfig.expertFormat : null;
|
|
1512
|
+
const moeExpertIntermediateSize = useMoE && moeConfig.expertIntermediateSize != null
|
|
1513
|
+
? Number(moeConfig.expertIntermediateSize)
|
|
1514
|
+
: resolvedIntermediateSize;
|
|
1515
|
+
if (
|
|
1516
|
+
useMoE
|
|
1517
|
+
&& expertFormat === 'gemma4'
|
|
1518
|
+
&& (!Number.isFinite(moeExpertIntermediateSize) || moeExpertIntermediateSize <= 0)
|
|
1519
|
+
) {
|
|
1520
|
+
throw new Error(`Manifest "${manifest.modelId}" has invalid moeConfig.expertIntermediateSize for Gemma-style experts.`);
|
|
1521
|
+
}
|
|
1301
1522
|
|
|
1302
1523
|
// RoPE scaling - use manifest inference as source of truth (not raw config)
|
|
1303
1524
|
const ropeScale = inf.rope.ropeScalingFactor;
|
|
@@ -1397,6 +1618,49 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1397
1618
|
return Math.trunc(num);
|
|
1398
1619
|
};
|
|
1399
1620
|
|
|
1621
|
+
const resolveTensorOutputRows = (tensorInfo, hiddenSize) => {
|
|
1622
|
+
const shape = Array.isArray(tensorInfo?.shape) ? tensorInfo.shape : null;
|
|
1623
|
+
if (!shape || shape.length < 2) return null;
|
|
1624
|
+
const dim0 = Number(shape[0]);
|
|
1625
|
+
const dim1 = Number(shape[1]);
|
|
1626
|
+
if (!Number.isFinite(dim0) || !Number.isFinite(dim1)) return null;
|
|
1627
|
+
if (dim1 === hiddenSize) return Math.trunc(dim0);
|
|
1628
|
+
if (dim0 === hiddenSize) return Math.trunc(dim1);
|
|
1629
|
+
return null;
|
|
1630
|
+
};
|
|
1631
|
+
|
|
1632
|
+
const deriveGlobalKVHeadsFromManifest = () => {
|
|
1633
|
+
if (!Array.isArray(layerTypes) || archGlobalHeadDim == null) {
|
|
1634
|
+
return null;
|
|
1635
|
+
}
|
|
1636
|
+
const tensors = manifest?.tensors && typeof manifest.tensors === 'object' ? manifest.tensors : null;
|
|
1637
|
+
if (!tensors) return null;
|
|
1638
|
+
const globalLayerIdx = layerTypes.findIndex((layerType) => {
|
|
1639
|
+
const normalized = typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
|
|
1640
|
+
return normalized && normalized !== 'sliding_attention' && normalized !== 'local_attention'
|
|
1641
|
+
&& normalized !== 'local' && normalized !== 'sliding';
|
|
1642
|
+
});
|
|
1643
|
+
if (globalLayerIdx < 0) return null;
|
|
1644
|
+
const layerMarker = `.layers.${globalLayerIdx}.`;
|
|
1645
|
+
for (const [tensorName, tensorInfo] of Object.entries(tensors)) {
|
|
1646
|
+
if (!tensorName.includes(layerMarker) || !tensorName.includes('.self_attn.k_proj.weight')) {
|
|
1647
|
+
continue;
|
|
1648
|
+
}
|
|
1649
|
+
const rows = resolveTensorOutputRows(tensorInfo, arch.hiddenSize);
|
|
1650
|
+
if (rows != null && rows % archGlobalHeadDim === 0) {
|
|
1651
|
+
return rows / archGlobalHeadDim;
|
|
1652
|
+
}
|
|
1653
|
+
}
|
|
1654
|
+
return null;
|
|
1655
|
+
};
|
|
1656
|
+
|
|
1657
|
+
const archNumGlobalKVHeads = parsePositiveInt(
|
|
1658
|
+
arch.numGlobalKeyValueHeads
|
|
1659
|
+
?? arch.numGlobalKVHeads
|
|
1660
|
+
?? config.num_global_key_value_heads
|
|
1661
|
+
?? config.num_global_kv_heads
|
|
1662
|
+
) ?? deriveGlobalKVHeadsFromManifest();
|
|
1663
|
+
|
|
1400
1664
|
const linearNumKeyHeads = parsePositiveInt(arch.linearNumKeyHeads ?? config.linear_num_key_heads);
|
|
1401
1665
|
const linearNumValueHeads = parsePositiveInt(arch.linearNumValueHeads ?? config.linear_num_value_heads);
|
|
1402
1666
|
const linearKeyHeadDim = parsePositiveInt(arch.linearKeyHeadDim ?? config.linear_key_head_dim);
|
|
@@ -1414,8 +1678,12 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1414
1678
|
? 'replay_prefill'
|
|
1415
1679
|
: 'incremental';
|
|
1416
1680
|
const perLayerInputsSession = resolvePerLayerInputsSession(inf, merged.modelId);
|
|
1681
|
+
const sessionSettings = resolveSessionSettings(inf, merged.modelId);
|
|
1682
|
+
const largeWeightsConfig = resolveLargeWeightsConfig(inf, merged.modelId);
|
|
1683
|
+
const diffusionGemma = inf.diffusionGemma ?? null;
|
|
1417
1684
|
|
|
1418
1685
|
return {
|
|
1686
|
+
modelType: manifest.modelType,
|
|
1419
1687
|
numLayers: arch.numLayers,
|
|
1420
1688
|
hiddenSize: arch.hiddenSize,
|
|
1421
1689
|
intermediateSize: resolvedIntermediateSize,
|
|
@@ -1423,6 +1691,7 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1423
1691
|
maxIntermediateSize,
|
|
1424
1692
|
numHeads: archNumHeads,
|
|
1425
1693
|
numKVHeads: archNumKVHeads,
|
|
1694
|
+
numGlobalKVHeads: archNumGlobalKVHeads,
|
|
1426
1695
|
headDim: archHeadDim,
|
|
1427
1696
|
globalHeadDim: archGlobalHeadDim,
|
|
1428
1697
|
vocabSize: arch.vocabSize,
|
|
@@ -1434,6 +1703,7 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1434
1703
|
numExperts,
|
|
1435
1704
|
moeTopK,
|
|
1436
1705
|
expertFormat,
|
|
1706
|
+
moeExpertIntermediateSize,
|
|
1437
1707
|
slidingWindow: inf.attention.slidingWindow,
|
|
1438
1708
|
ropeTheta: inf.rope.ropeTheta,
|
|
1439
1709
|
ropeLocalTheta: inf.rope.ropeLocalTheta,
|
|
@@ -1465,6 +1735,7 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1465
1735
|
embeddingVocabSize: inf.output.embeddingVocabSize,
|
|
1466
1736
|
embeddingPostprocessor,
|
|
1467
1737
|
hiddenActivation,
|
|
1738
|
+
ffnBranchMode: inf.ffn.branchMode ?? 'auto',
|
|
1468
1739
|
useDoubleWideMlp: inf.ffn.useDoubleWideMlp,
|
|
1469
1740
|
swigluLimit: inf.ffn.swigluLimit,
|
|
1470
1741
|
stopTokenIds,
|
|
@@ -1480,15 +1751,21 @@ export function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1480
1751
|
finalLogitSoftcapping: inf.output.finalLogitSoftcapping,
|
|
1481
1752
|
attnLogitSoftcapping: inf.attention.attnLogitSoftcapping,
|
|
1482
1753
|
queryKeyNorm: inf.attention.queryKeyNorm,
|
|
1754
|
+
queryKeyNormLayers: inf.attention.queryKeyNormLayers ?? null,
|
|
1755
|
+
queryKeyNormWeightLayers: inf.attention.queryKeyNormWeightLayers ?? null,
|
|
1483
1756
|
valueNorm: inf.attention.valueNorm,
|
|
1484
1757
|
attentionOutputGate: inf.attention.attentionOutputGate === true,
|
|
1758
|
+
outputGateType: inf.attention.outputGateType ?? null,
|
|
1485
1759
|
queryPreAttnScalar,
|
|
1486
1760
|
layerPipeline: inf.pipeline ?? null,
|
|
1487
1761
|
chatTemplateType,
|
|
1488
1762
|
chatTemplateEnabled,
|
|
1489
1763
|
chatTemplateThinking,
|
|
1490
1764
|
decodeStrategy,
|
|
1765
|
+
diffusionGemma,
|
|
1491
1766
|
perLayerInputsSession,
|
|
1767
|
+
sessionSettings,
|
|
1768
|
+
largeWeightsConfig,
|
|
1492
1769
|
kernelPath: null,
|
|
1493
1770
|
visionConfig: resolveVisionConfig(config, manifest),
|
|
1494
1771
|
audioConfig: resolveAudioConfig(config, manifest),
|
|
@@ -1546,6 +1823,8 @@ export function parseModelConfig(manifest, runtimeOverrides) {
|
|
|
1546
1823
|
);
|
|
1547
1824
|
}
|
|
1548
1825
|
|
|
1826
|
+
assertSupportedManifestInference(manifest);
|
|
1827
|
+
|
|
1549
1828
|
log.info('Config', 'Using manifest-first config (source of truth)');
|
|
1550
1829
|
return parseModelConfigFromManifest(manifest, runtimeOverrides);
|
|
1551
1830
|
}
|
|
@@ -2,30 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
import { readBuffer } from '../../../../memory/buffer-pool.js';
|
|
4
4
|
import { isBufferStatsEnabled } from './config.js';
|
|
5
|
+
import { f16ToF32 } from '../../../../loader/dtype-utils.js';
|
|
5
6
|
|
|
6
|
-
|
|
7
|
-
// Math Helpers
|
|
8
|
-
// ============================================================================
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
export function f16ToF32(h) {
|
|
12
|
-
const sign = (h >> 15) & 0x1;
|
|
13
|
-
const exp = (h >> 10) & 0x1f;
|
|
14
|
-
const mant = h & 0x3ff;
|
|
15
|
-
|
|
16
|
-
if (exp === 0) {
|
|
17
|
-
if (mant === 0) return sign ? -0 : 0;
|
|
18
|
-
const f = mant / 1024 * Math.pow(2, -14);
|
|
19
|
-
return sign ? -f : f;
|
|
20
|
-
}
|
|
21
|
-
if (exp === 31) {
|
|
22
|
-
return mant ? NaN : (sign ? -Infinity : Infinity);
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
const f = (1 + mant / 1024) * Math.pow(2, exp - 15);
|
|
26
|
-
return sign ? -f : f;
|
|
27
|
-
}
|
|
28
|
-
|
|
7
|
+
export { f16ToF32 };
|
|
29
8
|
|
|
30
9
|
export function decodeReadback(buffer, dtype) {
|
|
31
10
|
if (dtype === 'f32') {
|