doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -211,6 +211,17 @@ function resolveSessionSettings(inferenceConfig, modelId) {
|
|
|
211
211
|
`"${String(submit)}"; expected one of ${PREFILL_CHUNK_SUBMIT_MODES.join(', ')}.`
|
|
212
212
|
);
|
|
213
213
|
}
|
|
214
|
+
const tokenChunk = session?.prefillTokenChunkSize;
|
|
215
|
+
if (
|
|
216
|
+
tokenChunk !== undefined
|
|
217
|
+
&& tokenChunk !== null
|
|
218
|
+
&& (!Number.isInteger(tokenChunk) || tokenChunk <= 0)
|
|
219
|
+
) {
|
|
220
|
+
throw new Error(
|
|
221
|
+
`Manifest "${modelId}" has invalid inference.session.prefillTokenChunkSize ` +
|
|
222
|
+
`"${String(tokenChunk)}"; expected null or a positive integer.`
|
|
223
|
+
);
|
|
224
|
+
}
|
|
214
225
|
const flash = session?.useFlashPrefillAttention;
|
|
215
226
|
if (flash !== undefined && flash !== null && typeof flash !== 'boolean') {
|
|
216
227
|
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFlashPrefillAttention "${String(flash)}"; expected boolean.`);
|
|
@@ -219,14 +230,40 @@ function resolveSessionSettings(inferenceConfig, modelId) {
|
|
|
219
230
|
if (wide !== undefined && wide !== null && typeof wide !== 'boolean') {
|
|
220
231
|
throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileQ4KPrefill "${String(wide)}"; expected boolean.`);
|
|
221
232
|
}
|
|
233
|
+
const wideDecode = session?.useWideTileQ4KDecode;
|
|
234
|
+
if (wideDecode !== undefined && wideDecode !== null && typeof wideDecode !== 'boolean') {
|
|
235
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileQ4KDecode "${String(wideDecode)}"; expected boolean.`);
|
|
236
|
+
}
|
|
237
|
+
const sandwichRmsNormPair = session?.useSandwichRMSNormPairFusion;
|
|
238
|
+
if (sandwichRmsNormPair !== undefined && sandwichRmsNormPair !== null && typeof sandwichRmsNormPair !== 'boolean') {
|
|
239
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useSandwichRMSNormPairFusion "${String(sandwichRmsNormPair)}"; expected boolean.`);
|
|
240
|
+
}
|
|
241
|
+
const postFfnNextInputRmsNormPair = session?.usePostFfnNextInputRMSNormPairFusion;
|
|
242
|
+
if (postFfnNextInputRmsNormPair !== undefined && postFfnNextInputRmsNormPair !== null && typeof postFfnNextInputRmsNormPair !== 'boolean') {
|
|
243
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.usePostFfnNextInputRMSNormPairFusion "${String(postFfnNextInputRmsNormPair)}"; expected boolean.`);
|
|
244
|
+
}
|
|
245
|
+
const fusedQKVSplitQKNorm = session?.useFusedQKVSplitQKNorm;
|
|
246
|
+
if (fusedQKVSplitQKNorm !== undefined && fusedQKVSplitQKNorm !== null && typeof fusedQKVSplitQKNorm !== 'boolean') {
|
|
247
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFusedQKVSplitQKNorm "${String(fusedQKVSplitQKNorm)}"; expected boolean.`);
|
|
248
|
+
}
|
|
249
|
+
const fusedQKVSplitQKNormRoPE = session?.useFusedQKVSplitQKNormRoPE;
|
|
250
|
+
if (fusedQKVSplitQKNormRoPE !== undefined && fusedQKVSplitQKNormRoPE !== null && typeof fusedQKVSplitQKNormRoPE !== 'boolean') {
|
|
251
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFusedQKVSplitQKNormRoPE "${String(fusedQKVSplitQKNormRoPE)}"; expected boolean.`);
|
|
252
|
+
}
|
|
222
253
|
const retain = session?.retainQ4KMaterialization;
|
|
223
254
|
if (retain !== undefined && retain !== null && typeof retain !== 'boolean') {
|
|
224
255
|
throw new Error(`Manifest "${modelId}" has invalid inference.session.retainQ4KMaterialization "${String(retain)}"; expected boolean.`);
|
|
225
256
|
}
|
|
226
257
|
return {
|
|
227
258
|
prefillChunkSubmitMode: submit ?? null,
|
|
259
|
+
prefillTokenChunkSize: tokenChunk ?? null,
|
|
228
260
|
useFlashPrefillAttention: flash ?? null,
|
|
229
261
|
useWideTileQ4KPrefill: wide ?? null,
|
|
262
|
+
useWideTileQ4KDecode: wideDecode ?? null,
|
|
263
|
+
useSandwichRMSNormPairFusion: sandwichRmsNormPair ?? null,
|
|
264
|
+
usePostFfnNextInputRMSNormPairFusion: postFfnNextInputRmsNormPair ?? null,
|
|
265
|
+
useFusedQKVSplitQKNorm: fusedQKVSplitQKNorm ?? null,
|
|
266
|
+
useFusedQKVSplitQKNormRoPE: fusedQKVSplitQKNormRoPE ?? null,
|
|
230
267
|
retainQ4KMaterialization: retain ?? null,
|
|
231
268
|
};
|
|
232
269
|
}
|
|
@@ -598,6 +635,33 @@ function hasManifestInference(manifest) {
|
|
|
598
635
|
return 'inference' in manifest && manifest.inference != null;
|
|
599
636
|
}
|
|
600
637
|
|
|
638
|
+
function normalizeUnsupportedText(value) {
|
|
639
|
+
if (typeof value !== 'string') return '';
|
|
640
|
+
return value.trim();
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
export function assertSupportedManifestInference(manifest) {
|
|
644
|
+
const modelId = manifest?.modelId ?? 'unknown';
|
|
645
|
+
const unsupported = manifest?.inference?.unsupported;
|
|
646
|
+
if (unsupported == null) {
|
|
647
|
+
return;
|
|
648
|
+
}
|
|
649
|
+
if (typeof unsupported !== 'object' || Array.isArray(unsupported)) {
|
|
650
|
+
throw new Error(
|
|
651
|
+
`Manifest "${modelId}" has invalid inference.unsupported. ` +
|
|
652
|
+
'Expected null or an object with code, message, and recommendation fields.'
|
|
653
|
+
);
|
|
654
|
+
}
|
|
655
|
+
const code = normalizeUnsupportedText(unsupported.code) || 'unsupported-manifest-contract';
|
|
656
|
+
const message = normalizeUnsupportedText(unsupported.message);
|
|
657
|
+
const recommendation = normalizeUnsupportedText(unsupported.recommendation);
|
|
658
|
+
throw new Error(
|
|
659
|
+
`Manifest "${modelId}" is not supported by Doppler runtime (${code}).` +
|
|
660
|
+
(message ? ` ${message}` : '') +
|
|
661
|
+
(recommendation ? ` ${recommendation}` : '')
|
|
662
|
+
);
|
|
663
|
+
}
|
|
664
|
+
|
|
601
665
|
|
|
602
666
|
export function validateRequiredInferenceFields(inf, modelId) {
|
|
603
667
|
inf = inf ?? {};
|
|
@@ -617,6 +681,36 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
617
681
|
if (inf.attention.queryKeyNorm == null) {
|
|
618
682
|
errors.push('attention.queryKeyNorm is required');
|
|
619
683
|
}
|
|
684
|
+
if (
|
|
685
|
+
inf.attention.queryKeyNormLayers !== undefined
|
|
686
|
+
&& inf.attention.queryKeyNormLayers !== null
|
|
687
|
+
) {
|
|
688
|
+
if (!Array.isArray(inf.attention.queryKeyNormLayers)) {
|
|
689
|
+
errors.push('attention.queryKeyNormLayers must be null or an array of layer indices');
|
|
690
|
+
} else {
|
|
691
|
+
for (const layerIdx of inf.attention.queryKeyNormLayers) {
|
|
692
|
+
if (!Number.isInteger(layerIdx) || layerIdx < 0) {
|
|
693
|
+
errors.push('attention.queryKeyNormLayers must contain only non-negative integer layer indices');
|
|
694
|
+
break;
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
if (
|
|
700
|
+
inf.attention.queryKeyNormWeightLayers !== undefined
|
|
701
|
+
&& inf.attention.queryKeyNormWeightLayers !== null
|
|
702
|
+
) {
|
|
703
|
+
if (!Array.isArray(inf.attention.queryKeyNormWeightLayers)) {
|
|
704
|
+
errors.push('attention.queryKeyNormWeightLayers must be null or an array of layer indices');
|
|
705
|
+
} else {
|
|
706
|
+
for (const layerIdx of inf.attention.queryKeyNormWeightLayers) {
|
|
707
|
+
if (!Number.isInteger(layerIdx) || layerIdx < 0) {
|
|
708
|
+
errors.push('attention.queryKeyNormWeightLayers must contain only non-negative integer layer indices');
|
|
709
|
+
break;
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
}
|
|
620
714
|
if (inf.attention.valueNorm == null) {
|
|
621
715
|
errors.push('attention.valueNorm is required');
|
|
622
716
|
}
|
|
@@ -658,6 +752,15 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
658
752
|
if (inf.ffn.gatedActivation == null) {
|
|
659
753
|
errors.push('ffn.gatedActivation is required');
|
|
660
754
|
}
|
|
755
|
+
if (inf.ffn.branchMode !== undefined) {
|
|
756
|
+
const normalizedBranchMode = typeof inf.ffn.branchMode === 'string'
|
|
757
|
+
? inf.ffn.branchMode.trim().toLowerCase()
|
|
758
|
+
: '';
|
|
759
|
+
const supportedBranchModes = new Set(['auto', 'dense', 'moe', 'dense_plus_moe']);
|
|
760
|
+
if (!supportedBranchModes.has(normalizedBranchMode)) {
|
|
761
|
+
errors.push('ffn.branchMode must be one of: auto, dense, moe, dense_plus_moe');
|
|
762
|
+
}
|
|
763
|
+
}
|
|
661
764
|
if (inf.ffn.useDoubleWideMlp == null) {
|
|
662
765
|
errors.push('ffn.useDoubleWideMlp is required');
|
|
663
766
|
}
|
|
@@ -921,6 +1024,17 @@ function resolveVisionConfig(rawConfig, manifest) {
|
|
|
921
1024
|
}
|
|
922
1025
|
return Math.trunc(number);
|
|
923
1026
|
};
|
|
1027
|
+
const resolveRequiredNonNegativeInteger = (keys, label) => {
|
|
1028
|
+
const value = resolveRequiredVisionField(keys, label);
|
|
1029
|
+
const number = Number(value);
|
|
1030
|
+
if (!Number.isFinite(number) || number < 0 || Math.floor(number) !== number) {
|
|
1031
|
+
throw new Error(
|
|
1032
|
+
`Manifest "${modelId}" has invalid vision_config.${label}=${JSON.stringify(value)}. ` +
|
|
1033
|
+
'Expected a non-negative integer.'
|
|
1034
|
+
);
|
|
1035
|
+
}
|
|
1036
|
+
return Math.trunc(number);
|
|
1037
|
+
};
|
|
924
1038
|
const resolveRequiredPositiveNumber = (value, label) => {
|
|
925
1039
|
const number = Number(value);
|
|
926
1040
|
if (!Number.isFinite(number) || number <= 0) {
|
|
@@ -940,14 +1054,21 @@ function resolveVisionConfig(rawConfig, manifest) {
|
|
|
940
1054
|
}
|
|
941
1055
|
|
|
942
1056
|
if (visionArchitecture === 'gemma4') {
|
|
1057
|
+
const depth = resolveRequiredNonNegativeInteger(['depth', 'num_hidden_layers'], 'num_hidden_layers');
|
|
1058
|
+
const isEncoderFree = (depth === 0);
|
|
1059
|
+
|
|
943
1060
|
const hiddenSize = resolveRequiredPositiveInteger(['hidden_size'], 'hidden_size');
|
|
944
|
-
const numHeads =
|
|
1061
|
+
const numHeads = isEncoderFree
|
|
1062
|
+
? (vc.num_heads !== undefined || vc.num_attention_heads !== undefined ? resolveRequiredPositiveInteger(['num_heads', 'num_attention_heads'], 'num_attention_heads') : 1)
|
|
1063
|
+
: resolveRequiredPositiveInteger(['num_heads', 'num_attention_heads'], 'num_attention_heads');
|
|
945
1064
|
const ropeParameters = vc.rope_parameters;
|
|
946
1065
|
if (!ropeParameters || typeof ropeParameters !== 'object') {
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
1066
|
+
if (!isEncoderFree) {
|
|
1067
|
+
throw new Error(
|
|
1068
|
+
`Manifest "${modelId}" is missing vision_config.rope_parameters. ` +
|
|
1069
|
+
'Re-convert the model with explicit Gemma 4 vision RoPE metadata.'
|
|
1070
|
+
);
|
|
1071
|
+
}
|
|
951
1072
|
}
|
|
952
1073
|
const hiddenActivation = String(resolveRequiredVisionField(['hidden_activation'], 'hidden_activation')).trim();
|
|
953
1074
|
if (hiddenActivation !== 'gelu' && hiddenActivation !== 'gelu_pytorch_tanh') {
|
|
@@ -968,12 +1089,18 @@ function resolveVisionConfig(rawConfig, manifest) {
|
|
|
968
1089
|
}
|
|
969
1090
|
|
|
970
1091
|
return {
|
|
971
|
-
depth
|
|
1092
|
+
depth,
|
|
972
1093
|
hiddenSize,
|
|
973
|
-
intermediateSize:
|
|
1094
|
+
intermediateSize: isEncoderFree
|
|
1095
|
+
? (vc.intermediate_size !== undefined ? resolveRequiredPositiveInteger(['intermediate_size'], 'intermediate_size') : 1)
|
|
1096
|
+
: resolveRequiredPositiveInteger(['intermediate_size'], 'intermediate_size'),
|
|
974
1097
|
numHeads,
|
|
975
|
-
numKeyValueHeads:
|
|
976
|
-
|
|
1098
|
+
numKeyValueHeads: isEncoderFree
|
|
1099
|
+
? (vc.num_key_value_heads !== undefined ? resolveRequiredPositiveInteger(['num_key_value_heads'], 'num_key_value_heads') : 1)
|
|
1100
|
+
: resolveRequiredPositiveInteger(['num_key_value_heads'], 'num_key_value_heads'),
|
|
1101
|
+
headDim: isEncoderFree
|
|
1102
|
+
? (vc.head_dim !== undefined || vc.global_head_dim !== undefined ? resolveRequiredPositiveInteger(['head_dim', 'global_head_dim'], 'head_dim') : 1)
|
|
1103
|
+
: resolveRequiredPositiveInteger(['head_dim', 'global_head_dim'], 'head_dim'),
|
|
977
1104
|
outHiddenSize: vc.out_hidden_size ?? vc.output_proj_dims ?? null,
|
|
978
1105
|
patchSize: resolveRequiredPositiveInteger(['patch_size'], 'patch_size'),
|
|
979
1106
|
poolingKernelSize: resolveRequiredPositiveInteger(['pooling_kernel_size'], 'pooling_kernel_size'),
|
|
@@ -981,7 +1108,9 @@ function resolveVisionConfig(rawConfig, manifest) {
|
|
|
981
1108
|
temporalPatchSize: vc.temporal_patch_size ?? null,
|
|
982
1109
|
positionEmbeddingSize: resolveRequiredPositiveInteger(['position_embedding_size'], 'position_embedding_size'),
|
|
983
1110
|
defaultOutputLength: resolveRequiredPositiveInteger(['default_output_length'], 'default_output_length'),
|
|
984
|
-
ropeTheta:
|
|
1111
|
+
ropeTheta: isEncoderFree && (!ropeParameters || ropeParameters.rope_theta === undefined)
|
|
1112
|
+
? 10000
|
|
1113
|
+
: resolveRequiredPositiveNumber(ropeParameters.rope_theta, 'vision_config.rope_parameters.rope_theta'),
|
|
985
1114
|
eps: resolveRequiredPositiveNumber(
|
|
986
1115
|
resolveRequiredVisionField(['eps', 'rms_norm_eps'], 'rms_norm_eps'),
|
|
987
1116
|
'vision_config.rms_norm_eps'
|
|
@@ -1079,6 +1208,16 @@ function resolveAudioConfig(rawConfig, manifest) {
|
|
|
1079
1208
|
}
|
|
1080
1209
|
return Math.trunc(number);
|
|
1081
1210
|
};
|
|
1211
|
+
const resolveRequiredNonNegativeInteger = (value, label) => {
|
|
1212
|
+
const number = Number(value);
|
|
1213
|
+
if (!Number.isFinite(number) || number < 0 || Math.floor(number) !== number) {
|
|
1214
|
+
throw new Error(
|
|
1215
|
+
`Manifest "${modelId}" has invalid audio_config.${label}=${JSON.stringify(value)}. ` +
|
|
1216
|
+
'Expected a non-negative integer.'
|
|
1217
|
+
);
|
|
1218
|
+
}
|
|
1219
|
+
return Math.trunc(number);
|
|
1220
|
+
};
|
|
1082
1221
|
const resolveRequiredPositiveNumber = (value, label) => {
|
|
1083
1222
|
const number = Number(value);
|
|
1084
1223
|
if (!Number.isFinite(number) || number <= 0) {
|
|
@@ -1097,31 +1236,50 @@ function resolveAudioConfig(rawConfig, manifest) {
|
|
|
1097
1236
|
);
|
|
1098
1237
|
}
|
|
1099
1238
|
|
|
1239
|
+
const depth = resolveRequiredNonNegativeInteger(ac.num_hidden_layers, 'num_hidden_layers');
|
|
1240
|
+
const isEncoderFree = (depth === 0);
|
|
1241
|
+
|
|
1100
1242
|
const hiddenSize = resolveRequiredPositiveInteger(ac.hidden_size, 'hidden_size');
|
|
1101
|
-
const numAttentionHeads =
|
|
1243
|
+
const numAttentionHeads = isEncoderFree
|
|
1244
|
+
? (ac.num_attention_heads !== undefined ? resolveRequiredPositiveInteger(ac.num_attention_heads, 'num_attention_heads') : 1)
|
|
1245
|
+
: resolveRequiredPositiveInteger(ac.num_attention_heads, 'num_attention_heads');
|
|
1102
1246
|
const headDim = Math.trunc(hiddenSize / numAttentionHeads);
|
|
1103
1247
|
|
|
1104
|
-
if (!
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1248
|
+
if (!isEncoderFree) {
|
|
1249
|
+
if (!Array.isArray(ac.subsampling_conv_channels) || ac.subsampling_conv_channels.length < 1) {
|
|
1250
|
+
throw new Error(
|
|
1251
|
+
`Manifest "${modelId}" is missing audio_config.subsampling_conv_channels array.`
|
|
1252
|
+
);
|
|
1253
|
+
}
|
|
1108
1254
|
}
|
|
1109
1255
|
|
|
1110
1256
|
return {
|
|
1111
1257
|
audioArchitecture,
|
|
1112
|
-
depth
|
|
1258
|
+
depth,
|
|
1113
1259
|
hiddenSize,
|
|
1114
1260
|
numAttentionHeads,
|
|
1115
1261
|
headDim,
|
|
1116
|
-
convKernelSize:
|
|
1117
|
-
|
|
1262
|
+
convKernelSize: isEncoderFree
|
|
1263
|
+
? (ac.conv_kernel_size !== undefined ? resolveRequiredPositiveInteger(ac.conv_kernel_size, 'conv_kernel_size') : 1)
|
|
1264
|
+
: resolveRequiredPositiveInteger(ac.conv_kernel_size, 'conv_kernel_size'),
|
|
1265
|
+
subsamplingConvChannels: isEncoderFree
|
|
1266
|
+
? (ac.subsampling_conv_channels ? ac.subsampling_conv_channels.map(Number) : [])
|
|
1267
|
+
: ac.subsampling_conv_channels.map(Number),
|
|
1118
1268
|
outputProjDims: resolveRequiredPositiveInteger(ac.output_proj_dims, 'output_proj_dims'),
|
|
1119
|
-
attentionContextLeft:
|
|
1269
|
+
attentionContextLeft: isEncoderFree
|
|
1270
|
+
? (ac.attention_context_left !== undefined ? resolveRequiredPositiveInteger(ac.attention_context_left, 'attention_context_left') : 1)
|
|
1271
|
+
: resolveRequiredPositiveInteger(ac.attention_context_left, 'attention_context_left'),
|
|
1120
1272
|
attentionContextRight: Number(ac.attention_context_right ?? 0),
|
|
1121
|
-
attentionChunkSize:
|
|
1122
|
-
|
|
1273
|
+
attentionChunkSize: isEncoderFree
|
|
1274
|
+
? (ac.attention_chunk_size !== undefined ? resolveRequiredPositiveInteger(ac.attention_chunk_size, 'attention_chunk_size') : 1)
|
|
1275
|
+
: resolveRequiredPositiveInteger(ac.attention_chunk_size, 'attention_chunk_size'),
|
|
1276
|
+
attentionLogitCap: isEncoderFree
|
|
1277
|
+
? (ac.attention_logit_cap !== undefined ? resolveRequiredPositiveNumber(ac.attention_logit_cap, 'attention_logit_cap') : 1.0)
|
|
1278
|
+
: resolveRequiredPositiveNumber(ac.attention_logit_cap, 'attention_logit_cap'),
|
|
1123
1279
|
attentionInvalidLogitsValue: Number(ac.attention_invalid_logits_value ?? -1e9),
|
|
1124
|
-
residualWeight:
|
|
1280
|
+
residualWeight: isEncoderFree
|
|
1281
|
+
? (ac.residual_weight !== undefined ? resolveRequiredPositiveNumber(ac.residual_weight, 'residual_weight') : 1.0)
|
|
1282
|
+
: resolveRequiredPositiveNumber(ac.residual_weight, 'residual_weight'),
|
|
1125
1283
|
rmsNormEps: resolveRequiredPositiveNumber(ac.rms_norm_eps ?? 1e-6, 'rms_norm_eps'),
|
|
1126
1284
|
hiddenAct: String(ac.hidden_act ?? 'silu').trim(),
|
|
1127
1285
|
useClippedLinears: ac.use_clipped_linears === true,
|
|
@@ -1351,6 +1509,16 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1351
1509
|
const numExperts = useMoE ? moeConfig.numExperts : 0;
|
|
1352
1510
|
const moeTopK = useMoE ? moeConfig.numExpertsPerToken : 0;
|
|
1353
1511
|
const expertFormat = useMoE ? moeConfig.expertFormat : null;
|
|
1512
|
+
const moeExpertIntermediateSize = useMoE && moeConfig.expertIntermediateSize != null
|
|
1513
|
+
? Number(moeConfig.expertIntermediateSize)
|
|
1514
|
+
: resolvedIntermediateSize;
|
|
1515
|
+
if (
|
|
1516
|
+
useMoE
|
|
1517
|
+
&& expertFormat === 'gemma4'
|
|
1518
|
+
&& (!Number.isFinite(moeExpertIntermediateSize) || moeExpertIntermediateSize <= 0)
|
|
1519
|
+
) {
|
|
1520
|
+
throw new Error(`Manifest "${manifest.modelId}" has invalid moeConfig.expertIntermediateSize for Gemma-style experts.`);
|
|
1521
|
+
}
|
|
1354
1522
|
|
|
1355
1523
|
// RoPE scaling - use manifest inference as source of truth (not raw config)
|
|
1356
1524
|
const ropeScale = inf.rope.ropeScalingFactor;
|
|
@@ -1512,8 +1680,10 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1512
1680
|
const perLayerInputsSession = resolvePerLayerInputsSession(inf, merged.modelId);
|
|
1513
1681
|
const sessionSettings = resolveSessionSettings(inf, merged.modelId);
|
|
1514
1682
|
const largeWeightsConfig = resolveLargeWeightsConfig(inf, merged.modelId);
|
|
1683
|
+
const diffusionGemma = inf.diffusionGemma ?? null;
|
|
1515
1684
|
|
|
1516
1685
|
return {
|
|
1686
|
+
modelType: manifest.modelType,
|
|
1517
1687
|
numLayers: arch.numLayers,
|
|
1518
1688
|
hiddenSize: arch.hiddenSize,
|
|
1519
1689
|
intermediateSize: resolvedIntermediateSize,
|
|
@@ -1533,6 +1703,7 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1533
1703
|
numExperts,
|
|
1534
1704
|
moeTopK,
|
|
1535
1705
|
expertFormat,
|
|
1706
|
+
moeExpertIntermediateSize,
|
|
1536
1707
|
slidingWindow: inf.attention.slidingWindow,
|
|
1537
1708
|
ropeTheta: inf.rope.ropeTheta,
|
|
1538
1709
|
ropeLocalTheta: inf.rope.ropeLocalTheta,
|
|
@@ -1564,6 +1735,7 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1564
1735
|
embeddingVocabSize: inf.output.embeddingVocabSize,
|
|
1565
1736
|
embeddingPostprocessor,
|
|
1566
1737
|
hiddenActivation,
|
|
1738
|
+
ffnBranchMode: inf.ffn.branchMode ?? 'auto',
|
|
1567
1739
|
useDoubleWideMlp: inf.ffn.useDoubleWideMlp,
|
|
1568
1740
|
swigluLimit: inf.ffn.swigluLimit,
|
|
1569
1741
|
stopTokenIds,
|
|
@@ -1579,6 +1751,8 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1579
1751
|
finalLogitSoftcapping: inf.output.finalLogitSoftcapping,
|
|
1580
1752
|
attnLogitSoftcapping: inf.attention.attnLogitSoftcapping,
|
|
1581
1753
|
queryKeyNorm: inf.attention.queryKeyNorm,
|
|
1754
|
+
queryKeyNormLayers: inf.attention.queryKeyNormLayers ?? null,
|
|
1755
|
+
queryKeyNormWeightLayers: inf.attention.queryKeyNormWeightLayers ?? null,
|
|
1582
1756
|
valueNorm: inf.attention.valueNorm,
|
|
1583
1757
|
attentionOutputGate: inf.attention.attentionOutputGate === true,
|
|
1584
1758
|
outputGateType: inf.attention.outputGateType ?? null,
|
|
@@ -1588,6 +1762,7 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1588
1762
|
chatTemplateEnabled,
|
|
1589
1763
|
chatTemplateThinking,
|
|
1590
1764
|
decodeStrategy,
|
|
1765
|
+
diffusionGemma,
|
|
1591
1766
|
perLayerInputsSession,
|
|
1592
1767
|
sessionSettings,
|
|
1593
1768
|
largeWeightsConfig,
|
|
@@ -1648,6 +1823,8 @@ export function parseModelConfig(manifest, runtimeOverrides) {
|
|
|
1648
1823
|
);
|
|
1649
1824
|
}
|
|
1650
1825
|
|
|
1826
|
+
assertSupportedManifestInference(manifest);
|
|
1827
|
+
|
|
1651
1828
|
log.info('Config', 'Using manifest-first config (source of truth)');
|
|
1652
1829
|
return parseModelConfigFromManifest(manifest, runtimeOverrides);
|
|
1653
1830
|
}
|
|
@@ -25,7 +25,7 @@ export interface ImplicitDtypeTransitionOptions {
|
|
|
25
25
|
toDtype?: 'f16' | 'f32' | null;
|
|
26
26
|
op?: string | null;
|
|
27
27
|
detail?: string | null;
|
|
28
|
-
transitionDeclaredBy?: 'step_precision' | null;
|
|
28
|
+
transitionDeclaredBy?: 'step_precision' | 'explicit_cast_step' | null;
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
/**
|
|
@@ -102,7 +102,10 @@ export function assertImplicitDtypeTransitionAllowed(options = {}) {
|
|
|
102
102
|
if (policy !== 'require_cast_step') {
|
|
103
103
|
return;
|
|
104
104
|
}
|
|
105
|
-
if (
|
|
105
|
+
if (
|
|
106
|
+
options.transitionDeclaredBy === 'step_precision'
|
|
107
|
+
|| options.transitionDeclaredBy === 'explicit_cast_step'
|
|
108
|
+
) {
|
|
106
109
|
return;
|
|
107
110
|
}
|
|
108
111
|
|
|
@@ -6,7 +6,7 @@ import type { CommandRecorder } from '../../../gpu/command-recorder.js';
|
|
|
6
6
|
import type { ProbeConfigSchema } from '../../../config/schema/index.js';
|
|
7
7
|
import type { ExecutionV1PoliciesSchema } from '../../../config/schema/execution-v1.schema.js';
|
|
8
8
|
import type { Tensor } from '../../../gpu/tensor.js';
|
|
9
|
-
import type { CpuWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
9
|
+
import type { CpuWeightBuffer, SplitWeightBuffer, WeightStorageEncoding } from '../../../gpu/weight-buffer.js';
|
|
10
10
|
|
|
11
11
|
export interface EmbedConfig {
|
|
12
12
|
hiddenSize: number;
|
|
@@ -20,7 +20,8 @@ export interface EmbedConfig {
|
|
|
20
20
|
indexOffset?: number;
|
|
21
21
|
transpose?: boolean;
|
|
22
22
|
activationDtype?: 'f16' | 'f32';
|
|
23
|
-
embeddingDtype?: 'f16' | 'f32';
|
|
23
|
+
embeddingDtype?: 'f16' | 'f32' | 'litert_int4';
|
|
24
|
+
embeddingStorageEncoding?: WeightStorageEncoding | null;
|
|
24
25
|
executionPolicies?: ExecutionV1PoliciesSchema | null;
|
|
25
26
|
probeStage?: string;
|
|
26
27
|
inputHiddenSize?: number;
|
|
@@ -38,7 +39,7 @@ export interface ValidationResult {
|
|
|
38
39
|
|
|
39
40
|
export function embed(
|
|
40
41
|
tokenIds: number[] | Uint32Array | GPUBuffer,
|
|
41
|
-
embedBuffer: GPUBuffer | Float32Array | CpuWeightBuffer,
|
|
42
|
+
embedBuffer: GPUBuffer | Float32Array | CpuWeightBuffer | SplitWeightBuffer,
|
|
42
43
|
config: EmbedConfig
|
|
43
44
|
): Promise<Tensor>;
|
|
44
45
|
|
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
import { getDevice, getKernelCapabilities } from '../../../gpu/device.js';
|
|
4
4
|
import { acquireBuffer, releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
|
|
5
|
-
import { runGather, recordGather, runScale, recordScale } from '../../../gpu/kernel-selector.js';
|
|
5
|
+
import { runGather, recordGather, runGatherSplit, recordGatherSplit, runScale, recordScale } from '../../../gpu/kernel-selector.js';
|
|
6
6
|
import { log, trace } from '../../../debug/index.js';
|
|
7
7
|
import { runProbes } from './probes.js';
|
|
8
8
|
import { decodeReadback } from './debug-utils/index.js';
|
|
9
9
|
import { createTensor } from '../../../gpu/tensor.js';
|
|
10
10
|
import { castF32ToF16, recordCastF32ToF16 } from '../../../gpu/kernels/cast.js';
|
|
11
|
-
import { isCpuWeightBuffer, isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
|
|
11
|
+
import { isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
12
12
|
import { f16ToF32 } from '../../../loader/dtype-utils.js';
|
|
13
13
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
14
14
|
|
|
@@ -88,6 +88,7 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
88
88
|
inputHiddenSize = hiddenSize,
|
|
89
89
|
hiddenOffset = 0,
|
|
90
90
|
stats = null,
|
|
91
|
+
embeddingStorageEncoding = null,
|
|
91
92
|
} = config;
|
|
92
93
|
const device = getDevice();
|
|
93
94
|
const tokenBufferInput = isGpuBufferInstance(tokenIds);
|
|
@@ -324,16 +325,25 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
324
325
|
transpose,
|
|
325
326
|
outputDtype: selectRuleValue('shared', 'dtype', 'f16OrF32', { useF16 }),
|
|
326
327
|
embeddingDtype,
|
|
328
|
+
storageEncoding: embeddingStorageEncoding,
|
|
327
329
|
indexOffset,
|
|
328
330
|
inputHiddenSize,
|
|
329
331
|
hiddenOffset,
|
|
330
332
|
};
|
|
331
|
-
if (!isGpuBufferInstance(embedBuffer)) {
|
|
333
|
+
if (!isGpuBufferInstance(embedBuffer) && !isSplitWeightBuffer(embedBuffer)) {
|
|
332
334
|
throw new Error('[Embed] GPU embeddings required for gather path.');
|
|
333
335
|
}
|
|
334
|
-
const gatherOutput =
|
|
335
|
-
?
|
|
336
|
-
|
|
336
|
+
const gatherOutput = isSplitWeightBuffer(embedBuffer)
|
|
337
|
+
? (
|
|
338
|
+
recorder
|
|
339
|
+
? await recordGatherSplit(recorder, tokenIdBuffer, embedBuffer, numTokens, hiddenSize, vocabSize, gatherOptions)
|
|
340
|
+
: await runGatherSplit(tokenIdBuffer, embedBuffer, numTokens, hiddenSize, vocabSize, gatherOptions)
|
|
341
|
+
)
|
|
342
|
+
: (
|
|
343
|
+
recorder
|
|
344
|
+
? await recordGather(recorder, tokenIdBuffer, embedBuffer, numTokens, hiddenSize, vocabSize, gatherOptions)
|
|
345
|
+
: await runGather(tokenIdBuffer, embedBuffer, numTokens, hiddenSize, vocabSize, gatherOptions)
|
|
346
|
+
);
|
|
337
347
|
|
|
338
348
|
// Debug: Verify first token embedding
|
|
339
349
|
if (debug && !recorder && tokenIdArray && tokenIdArray.length > 0) {
|
|
@@ -20,6 +20,7 @@ export interface StaticExecutionPlan {
|
|
|
20
20
|
defaultStopCheckMode: 'batch' | 'per-token';
|
|
21
21
|
defaultMaxTokens: number;
|
|
22
22
|
readbackInterval: number | null;
|
|
23
|
+
readbackMode: 'sequential' | 'overlapped' | 'auto';
|
|
23
24
|
ringTokens: number | null;
|
|
24
25
|
ringStop: number | null;
|
|
25
26
|
ringStaging: number | null;
|
|
@@ -51,6 +52,7 @@ export interface ExecutionSessionPlan {
|
|
|
51
52
|
stopCheckMode: 'batch' | 'per-token';
|
|
52
53
|
maxTokens: number;
|
|
53
54
|
readbackInterval: number | null;
|
|
55
|
+
readbackMode: 'sequential' | 'overlapped' | 'auto';
|
|
54
56
|
ringTokens: number | null;
|
|
55
57
|
ringStop: number | null;
|
|
56
58
|
ringStaging: number | null;
|
|
@@ -122,6 +124,12 @@ export declare function resolveMaxBatchDecodeTokens(config: {
|
|
|
122
124
|
hasHotVocabularyBatchDecode?: boolean;
|
|
123
125
|
hasGpuSplitPerLayerInputs?: boolean;
|
|
124
126
|
hasLinearAttentionLayers?: boolean;
|
|
127
|
+
modelId?: string;
|
|
128
|
+
activationDtype?: string;
|
|
129
|
+
currentSeqLen?: number;
|
|
130
|
+
maxDecodeTokens?: number;
|
|
131
|
+
numLayers?: number;
|
|
132
|
+
hiddenSize?: number;
|
|
125
133
|
}): number | null;
|
|
126
134
|
|
|
127
135
|
export declare function resolvePrefillRecorderChunkLayers(config: {
|
|
@@ -432,6 +432,12 @@ export function resolveMaxBatchDecodeTokens(config) {
|
|
|
432
432
|
hasHotVocabularyBatchDecode: config.hasHotVocabularyBatchDecode === true,
|
|
433
433
|
hasGpuSplitPerLayerInputs: config.hasGpuSplitPerLayerInputs === true,
|
|
434
434
|
hasLinearAttentionLayers: config.hasLinearAttentionLayers === true,
|
|
435
|
+
modelId: typeof config.modelId === 'string' ? config.modelId : '',
|
|
436
|
+
activationDtype: typeof config.activationDtype === 'string' ? config.activationDtype : '',
|
|
437
|
+
currentSeqLen: Number.isFinite(config.currentSeqLen) ? config.currentSeqLen : 0,
|
|
438
|
+
maxDecodeTokens: Number.isFinite(config.maxDecodeTokens) ? config.maxDecodeTokens : 0,
|
|
439
|
+
numLayers: Number.isFinite(config.numLayers) ? config.numLayers : 0,
|
|
440
|
+
hiddenSize: Number.isFinite(config.hiddenSize) ? config.hiddenSize : 0,
|
|
435
441
|
});
|
|
436
442
|
if (value == null) {
|
|
437
443
|
return null;
|
|
@@ -7,6 +7,10 @@ export declare function requireSessionActivationDtype(
|
|
|
7
7
|
session: Record<string, unknown> | null | undefined,
|
|
8
8
|
label?: string
|
|
9
9
|
): 'f16' | 'f32';
|
|
10
|
+
export declare function requireSessionKVDtype(
|
|
11
|
+
session: Record<string, unknown> | null | undefined,
|
|
12
|
+
label?: string
|
|
13
|
+
): 'f16' | 'f32';
|
|
10
14
|
|
|
11
15
|
export declare function buildInlineKernelPath(
|
|
12
16
|
steps: readonly Record<string, unknown>[],
|
|
@@ -46,6 +46,17 @@ export function requireSessionActivationDtype(
|
|
|
46
46
|
return normalizeDtype(activationDtype, label);
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
export function requireSessionKVDtype(
|
|
50
|
+
session,
|
|
51
|
+
label = 'session.kvcache.kvDtype'
|
|
52
|
+
) {
|
|
53
|
+
const kvDtype = session?.kvcache?.kvDtype;
|
|
54
|
+
if (kvDtype == null) {
|
|
55
|
+
throw new Error(`[Execution] ${label} is required.`);
|
|
56
|
+
}
|
|
57
|
+
return normalizeDtype(kvDtype, label);
|
|
58
|
+
}
|
|
59
|
+
|
|
49
60
|
function toKernelPathStep(step) {
|
|
50
61
|
if (step.op === 'cast') return null;
|
|
51
62
|
if (!step.kernel) {
|
|
@@ -122,7 +133,7 @@ export function assertKernelPathSessionCompatibility(path, session) {
|
|
|
122
133
|
'inlineKernelPath.activationDtype'
|
|
123
134
|
);
|
|
124
135
|
const kvDtype = normalizeDtype(
|
|
125
|
-
path.kvDtype ?? session
|
|
136
|
+
path.kvDtype ?? requireSessionKVDtype(session),
|
|
126
137
|
'inlineKernelPath.kvDtype'
|
|
127
138
|
);
|
|
128
139
|
|
|
@@ -176,10 +187,7 @@ export function buildInlineKernelPath(
|
|
|
176
187
|
finitenessFallbackKernelPathId = null
|
|
177
188
|
) {
|
|
178
189
|
const activationDtype = requireSessionActivationDtype(session);
|
|
179
|
-
const kvDtype =
|
|
180
|
-
session?.kvcache?.kvDtype ?? activationDtype,
|
|
181
|
-
'session.kvcache.kvDtype'
|
|
182
|
-
);
|
|
190
|
+
const kvDtype = requireSessionKVDtype(session);
|
|
183
191
|
const decodeSteps = buildLayerPhaseSteps(steps, 'decode', 0);
|
|
184
192
|
const prefillSteps = buildLayerPhaseSteps(steps, 'prefill', 0);
|
|
185
193
|
if (decodeSteps.length === 0 && prefillSteps.length === 0) {
|
|
@@ -299,29 +307,37 @@ export function buildLayerPipelineFromExecution(steps, options = {}) {
|
|
|
299
307
|
}
|
|
300
308
|
|
|
301
309
|
const layerSteps = layerSectionSteps
|
|
302
|
-
.map((step) =>
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
? {
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
310
|
+
.map((step) => {
|
|
311
|
+
if (typeof step.src !== 'string' || step.src.length === 0) {
|
|
312
|
+
throw new Error(`[Execution] ${step.op}.src is required for execution-v1 layer steps.`);
|
|
313
|
+
}
|
|
314
|
+
if (typeof step.dst !== 'string' || step.dst.length === 0) {
|
|
315
|
+
throw new Error(`[Execution] ${step.op}.dst is required for execution-v1 layer steps.`);
|
|
316
|
+
}
|
|
317
|
+
return {
|
|
318
|
+
op: step.op,
|
|
319
|
+
phase: step.phase,
|
|
320
|
+
src: step.src,
|
|
321
|
+
dst: step.dst,
|
|
322
|
+
...(step.residual !== undefined ? { residual: step.residual } : {}),
|
|
323
|
+
...(step.a !== undefined ? { a: step.a } : {}),
|
|
324
|
+
...(step.b !== undefined ? { b: step.b } : {}),
|
|
325
|
+
...(step.variant !== undefined ? { variant: step.variant } : {}),
|
|
326
|
+
...(step.skipInputNorm !== undefined ? { skipInputNorm: step.skipInputNorm } : {}),
|
|
327
|
+
...(step.precision?.inputDtype
|
|
328
|
+
? { inputDtype: step.precision.inputDtype }
|
|
329
|
+
: (step.op === 'ffn' && ffnDtypeFallback ? { inputDtype: ffnDtypeFallback } : {})),
|
|
330
|
+
...(step.precision?.outputDtype
|
|
331
|
+
? { outputDtype: step.precision.outputDtype }
|
|
332
|
+
: (step.op === 'ffn' && ffnDtypeFallback ? { outputDtype: ffnDtypeFallback } : {})),
|
|
333
|
+
...(step.precision?.kvDtype ? { kvDtype: step.precision.kvDtype } : {}),
|
|
334
|
+
...(step.fromDtype ? { fromDtype: step.fromDtype } : {}),
|
|
335
|
+
...(step.toDtype ? { toDtype: step.toDtype } : {}),
|
|
336
|
+
...(step.probeStage ? { probeStage: step.probeStage } : {}),
|
|
337
|
+
...(step.name ? { name: step.name } : {}),
|
|
338
|
+
...(step.weight ? { weight: step.weight } : {}),
|
|
339
|
+
};
|
|
340
|
+
});
|
|
325
341
|
|
|
326
342
|
return {
|
|
327
343
|
steps: layerSteps,
|
|
@@ -49,9 +49,11 @@ export declare function compileExecutionV1(options?: {
|
|
|
49
49
|
};
|
|
50
50
|
modelId?: string;
|
|
51
51
|
numLayers?: number;
|
|
52
|
+
headDim?: number | null;
|
|
52
53
|
runtimeSession?: ExecutionV1SessionSchema | null;
|
|
53
54
|
runtimeCompute?: Record<string, unknown> | null;
|
|
54
55
|
kernelPathPolicy?: Record<string, unknown> | null;
|
|
56
|
+
executionPatch?: Record<string, unknown> | null;
|
|
55
57
|
capabilities?: {
|
|
56
58
|
hasSubgroups?: boolean;
|
|
57
59
|
hasF16?: boolean;
|