doppler-gpu 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +210 -0
- package/README.md +63 -38
- package/assets/doppler.svg +57 -1601
- package/package.json +111 -32
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +486 -0
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +885 -744
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +469 -43
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +66 -10
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +69 -4
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +34 -50
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +35 -2
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +148 -3
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +8 -60
- package/src/client/wrap-pipeline-handle.d.ts +82 -0
- package/src/client/wrap-pipeline-handle.js +259 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +28 -157
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +163 -126
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +3593 -1146
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +3 -1
- package/src/config/merge-helpers.js +2 -1
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +65 -7
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +20 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +16 -0
- package/src/config/schema/converter.schema.js +11 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +13 -2
- package/src/config/schema/doppler.schema.js +140 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +39 -1
- package/src/config/schema/execution-v1.schema.js +35 -20
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +14 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +175 -2
- package/src/config/schema/manifest.schema.js +58 -3
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -9
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +183 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
- package/src/config/transforms/execution-graph-transforms.js +829 -168
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +31 -14
- package/src/converter/core.d.ts +65 -0
- package/src/converter/core.js +1153 -22
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +218 -5
- package/src/converter/quantizer.d.ts +24 -4
- package/src/converter/quantizer.js +92 -2
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +296 -38
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1733 -58
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +5 -14
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +192 -12
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +111 -2
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +10 -0
- package/src/formats/rdrr/index.js +10 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +26 -6
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +193 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +111 -2
- package/src/formats/rdrr/validation.js +260 -1
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +45 -23
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +130 -14
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +1 -8
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +458 -16
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +13 -120
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +84 -8
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +25 -22
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +77 -5
- package/src/gpu/kernels/index.js +55 -4
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +44 -2
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +172 -10
- package/src/gpu/kernels/matmul.d.ts +1 -29
- package/src/gpu/kernels/matmul.js +240 -38
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +1 -56
- package/src/gpu/kernels/moe.js +46 -33
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -12
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +6 -4
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +30 -7
- package/src/gpu/kernels/rmsnorm.js +49 -8
- package/src/gpu/kernels/rmsnorm.wgsl +15 -12
- package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +28 -21
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +8 -223
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +43 -0
- package/src/gpu/kernels/utils.js +81 -11
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +10 -12
- package/src/gpu/uniform-cache.js +25 -30
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
- package/src/inference/browser-harness-suite-helpers.js +525 -3
- package/src/inference/browser-harness-text-helpers.d.ts +89 -0
- package/src/inference/browser-harness-text-helpers.js +459 -25
- package/src/inference/browser-harness.d.ts +7 -28
- package/src/inference/browser-harness.js +556 -79
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +31 -0
- package/src/inference/kv-cache/base.js +125 -2
- package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
- package/src/inference/kv-cache/mixed-geometry.js +121 -4
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
- package/src/inference/pipelines/text/attention/projections.js +398 -71
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +542 -56
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +343 -50
- package/src/inference/pipelines/text/attention/types.d.ts +7 -1
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +25 -26
- package/src/inference/pipelines/text/config.js +304 -25
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +25 -29
- package/src/inference/pipelines/text/embed.js +16 -45
- package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
- package/src/inference/pipelines/text/execution-plan.js +21 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
- package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
- package/src/inference/pipelines/text/execution-v1.js +461 -158
- package/src/inference/pipelines/text/ffn/dense.js +214 -6
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +84 -17
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +9 -7
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
- package/src/inference/pipelines/text/generator-steps.js +275 -79
- package/src/inference/pipelines/text/generator.d.ts +38 -5
- package/src/inference/pipelines/text/generator.js +1259 -496
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +13 -3
- package/src/inference/pipelines/text/init.js +213 -725
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +373 -558
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +419 -66
- package/src/inference/pipelines/text/logits/index.d.ts +3 -3
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +42 -4
- package/src/inference/pipelines/text/ops.js +20 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +119 -31
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +153 -9
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
- package/src/inference/pipelines/text/types.d.ts +92 -81
- package/src/inference/pipelines/text/weights.d.ts +4 -52
- package/src/inference/pipelines/text/weights.js +6 -56
- package/src/inference/pipelines/text.d.ts +33 -1
- package/src/inference/pipelines/text.js +375 -55
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +341 -174
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +14 -0
- package/src/loader/doppler-loader.js +378 -25
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +276 -11
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +61 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +52 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +36 -8
- package/src/loader/tensors/tensor-loader.js +521 -22
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +13 -42
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +57 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +162 -10
- package/src/rules/inference/capability-transforms.rules.json +111 -0
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +117 -16
- package/src/rules/inference/ffn.rules.json +24 -2
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +17 -0
- package/src/rules/rule-registry.js +59 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +241 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +31 -4
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +53 -9
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +8 -3
- package/src/tooling/command-api.d.ts +22 -1
- package/src/tooling/command-api.js +11 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +14 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +3 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +571 -37
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +107 -17
- package/src/tooling/node-command-runner.js +43 -2
- package/src/tooling/node-convert-worker-pool.js +15 -0
- package/src/tooling/node-convert-worker.js +11 -1
- package/src/tooling/node-converter.js +102 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +112 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +10 -5
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +110 -8
- package/src/tooling/source-runtime-bundle.d.ts +3 -0
- package/src/tooling/source-runtime-bundle.js +30 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +25 -12
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { cloneJsonValue as cloneConfigTree } from '../../utils/clone-json.js';
|
|
1
2
|
import { DEFAULT_LOADING_CONFIG } from './loading.schema.js';
|
|
2
3
|
import { DEFAULT_SHARED_RUNTIME_CONFIG } from './shared-runtime.schema.js';
|
|
3
4
|
import { DEFAULT_EMULATION_CONFIG, createEmulationConfig } from './emulation.schema.js';
|
|
@@ -29,6 +30,7 @@ export const DEFAULT_LARGE_WEIGHT_CONFIG = Object.freeze({
|
|
|
29
30
|
safetyRatio: 0.9,
|
|
30
31
|
preferF16: true,
|
|
31
32
|
lmHeadChunkRows: null,
|
|
33
|
+
gpuResidentOverrides: null,
|
|
32
34
|
});
|
|
33
35
|
|
|
34
36
|
export const DEFAULT_RUNTIME_CONFIG = {
|
|
@@ -43,21 +45,125 @@ export const DEFAULT_RUNTIME_CONFIG = {
|
|
|
43
45
|
repetitionPenalty: 1.1,
|
|
44
46
|
greedyThreshold: 0.01,
|
|
45
47
|
repetitionPenaltyWindow: 100,
|
|
48
|
+
suppressSpecialTokens: false,
|
|
49
|
+
suppressSpecialLikeTokens: false,
|
|
50
|
+
suppressTokenIds: [],
|
|
46
51
|
},
|
|
47
52
|
compute: {},
|
|
48
53
|
tokenizer: {},
|
|
49
54
|
largeWeights: DEFAULT_LARGE_WEIGHT_CONFIG,
|
|
50
55
|
kvcache: DEFAULT_KVCACHE_CONFIG,
|
|
51
56
|
diffusion: {},
|
|
57
|
+
diffusionGemma: {},
|
|
52
58
|
energy: {},
|
|
53
59
|
moe: DEFAULT_MOE_RUNTIME_CONFIG,
|
|
54
60
|
speculative: DEFAULT_SPECULATIVE_CONFIG,
|
|
55
61
|
generation: {
|
|
56
62
|
disableMultiTokenDecode: false,
|
|
57
63
|
},
|
|
64
|
+
kernelPathPolicy: mergeKernelPathPolicy(undefined, {
|
|
65
|
+
mode: 'capability-aware',
|
|
66
|
+
sourceScope: ['model', 'manifest', 'config'],
|
|
67
|
+
onIncompatible: 'remap',
|
|
68
|
+
}),
|
|
58
69
|
chatTemplate: DEFAULT_CHAT_TEMPLATE_CONFIG,
|
|
59
70
|
session: {
|
|
60
71
|
kvcache: DEFAULT_KVCACHE_CONFIG,
|
|
72
|
+
// "sync" (default): wait for each prefill chunk to finish on GPU before
|
|
73
|
+
// recording the next. Required when profile timings are being
|
|
74
|
+
// resolved and when low-memory constraints demand deterministic
|
|
75
|
+
// buffer release per chunk.
|
|
76
|
+
// "async": queue each prefill chunk without waiting — GPU queue ordering
|
|
77
|
+
// keeps commands correct and deferred cleanup releases tracked
|
|
78
|
+
// buffers when work completes. Reduces CPU-GPU roundtrips; profile
|
|
79
|
+
// paths override to sync automatically.
|
|
80
|
+
prefillChunkSubmitMode: 'sync',
|
|
81
|
+
// Number of transformer layers recorded per prefill command recorder
|
|
82
|
+
// before a chunk boundary fires. Larger values → fewer chunk
|
|
83
|
+
// transitions but longer intermediate-buffer lifetime. Default 4 is
|
|
84
|
+
// memory-conservative; profiles with known-safe memory can raise it.
|
|
85
|
+
prefillChunkLayers: 4,
|
|
86
|
+
prefillTokenChunkSize: null,
|
|
87
|
+
// Opt into the flash-attention prefill kernel (head_dim=256, f16 KV,
|
|
88
|
+
// contiguous layout). Two-pass dispatch raises RDNA3 workgroup
|
|
89
|
+
// occupancy by KV-axis splitting. The kernel itself enforces head_dim
|
|
90
|
+
// and layout preconditions at runtime. Default false until parity is
|
|
91
|
+
// validated on target hardware.
|
|
92
|
+
useFlashPrefillAttention: false,
|
|
93
|
+
// Opt into the fused gate + up projection + GeGLU activation prefill
|
|
94
|
+
// kernel (replaces gate_proj + up_proj + gelu activation dispatches
|
|
95
|
+
// with a single fused compute pass). Requires f16 weights + gelu
|
|
96
|
+
// activation + no LoRA on gate/up. Default false until parity is
|
|
97
|
+
// validated on target hardware.
|
|
98
|
+
useFusedGateUpGelu: false,
|
|
99
|
+
// Opt into the register-tiled Q4_K prefill matmul kernel (64x64 outputs
|
|
100
|
+
// per workgroup via 4x4 register tile per thread). Amortizes Q4_K
|
|
101
|
+
// dequantization across 16x the outputs of q4_fused_batched_f16a,
|
|
102
|
+
// replacing ~1024x too many launched workgroups on prefill shapes.
|
|
103
|
+
// Applied only to Q4_K-weight matmuls with f16 activations, f16 output,
|
|
104
|
+
// hasSubgroups, M>=16. Default false until parity is validated.
|
|
105
|
+
useTiledQ4KPrefill: false,
|
|
106
|
+
// Force the loader to retain Q4_K packed weights alongside the
|
|
107
|
+
// dequantized dense buffer ("mixed" materialization mode). Without a
|
|
108
|
+
// fused projection kernel declared in the execution graph, the default
|
|
109
|
+
// is "dense" which drops the Q4_K buffer after dequant — preventing the
|
|
110
|
+
// FFN fusion rule from firing its `hasQ4KMaterialization=true` branch.
|
|
111
|
+
// Memory cost: ~50% extra weight storage for Q4_K matmul weights.
|
|
112
|
+
retainQ4KMaterialization: false,
|
|
113
|
+
// Opt into the WideTile Q4_K prefill matmul (adapted from ORT
|
|
114
|
+
// MatMulNBitsWideTile). Register-tiled accumulation, 1 thread per
|
|
115
|
+
// output column, TILE_M rows accumulated simultaneously. Dramatically
|
|
116
|
+
// fewer workgroups launched vs q4_fused_batched_f16 at prefill shapes.
|
|
117
|
+
// Requires f32 activations + Q4_K weights + shader-f16 + M >= TILE_M.
|
|
118
|
+
// Default false until correctness + perf parity validated on target
|
|
119
|
+
// hardware.
|
|
120
|
+
useWideTileQ4KPrefill: false,
|
|
121
|
+
// Opt into the WideTile Q4_K decode matmul. This reuses the WideTile
|
|
122
|
+
// register kernel for one-row decode matmuls when explicitly enabled.
|
|
123
|
+
// Default false until correctness + perf parity validated on target
|
|
124
|
+
// hardware.
|
|
125
|
+
useWideTileQ4KDecode: false,
|
|
126
|
+
// Opt into a two-output sandwich RMSNorm decode kernel:
|
|
127
|
+
// post_attn_norm and pre_ffn_norm execute in one dispatch while still
|
|
128
|
+
// materializing both tensors for the downstream residual and FFN paths.
|
|
129
|
+
// Default false until correctness + perf validated.
|
|
130
|
+
useSandwichRMSNormPairFusion: false,
|
|
131
|
+
// Opt into a decode-only cross-layer RMSNorm pair:
|
|
132
|
+
// post_ffn_norm for layer N and input_norm for layer N+1 execute in one
|
|
133
|
+
// dispatch. The next layer consumes the precomputed input-norm tensor at
|
|
134
|
+
// its normal observation point. Default false until correctness + perf
|
|
135
|
+
// validated.
|
|
136
|
+
usePostFfnNextInputRMSNormPairFusion: false,
|
|
137
|
+
// Opt into the single-pass flash attention prefill kernel adapted from
|
|
138
|
+
// ORT's flash_attention.wgsl.template. 64 threads = 64 queries per WG;
|
|
139
|
+
// private Q/O tiles, shared K/V tiles, online softmax, no reduce pass.
|
|
140
|
+
// At Gemma 4 E2B prefill=64 this dispatches 8 WGs total (vs ~32 for the
|
|
141
|
+
// split+reduce Doppler flash path). Requires head_dim=256, f16 KV,
|
|
142
|
+
// contiguous KV layout, seqLen > 1. Supersedes useFlashPrefillAttention
|
|
143
|
+
// when both flags are set (this path is picked first). Default false.
|
|
144
|
+
useOrtFlashPrefillAttention: false,
|
|
145
|
+
// Opt into the WideTile Q4_K matmul + residual fusion at the ffn_down
|
|
146
|
+
// + ffn_residual call site (dense.js). Eliminates one separate residual
|
|
147
|
+
// dispatch per layer. Requires f32 activations + retained Q4_K weights
|
|
148
|
+
// plus an enabled WideTile phase.
|
|
149
|
+
// Default false until correctness + perf validated.
|
|
150
|
+
useWideTileResidualFusion: false,
|
|
151
|
+
// Opt into the RMSNorm + WideTile Q4_K matmul fusion at pre-matmul
|
|
152
|
+
// norm sites (input_norm→q/k/v_proj, pre_feedforward_norm→gate/up).
|
|
153
|
+
// Each fused call recomputes RMS internally (redundant across q/k/v
|
|
154
|
+
// but negligible) and skips the standalone rmsnorm dispatch upstream.
|
|
155
|
+
// Requires f32 activations + Q4_K weights retained + prefill. Default
|
|
156
|
+
// false until wiring lands and correctness validates.
|
|
157
|
+
useFusedRmsnormWideTile: false,
|
|
158
|
+
// Opt into fused packed-QKV split plus weighted Q/K RMSNorm. Replaces
|
|
159
|
+
// split_qkv + rmsnorm_qk when diagnostics do not need raw Q/K projection
|
|
160
|
+
// readbacks. Requires f32 QKV output and weighted Q/K norm.
|
|
161
|
+
useFusedQKVSplitQKNorm: false,
|
|
162
|
+
// Opt into fused packed-QKV split plus weighted Q/K RMSNorm and full-head
|
|
163
|
+
// non-interleaved RoPE. Replaces split_qkv + rmsnorm_qk + rope_qk when
|
|
164
|
+
// diagnostics do not need intermediate Q/K stage readbacks.
|
|
165
|
+
useFusedQKVSplitQKNormRoPE: false,
|
|
166
|
+
useGreedyLmHeadArgmaxFusion: false,
|
|
61
167
|
},
|
|
62
168
|
executionPatch: {},
|
|
63
169
|
},
|
|
@@ -73,13 +179,6 @@ export const DEFAULT_DOPPLER_CONFIG = {
|
|
|
73
179
|
runtime: DEFAULT_RUNTIME_CONFIG,
|
|
74
180
|
};
|
|
75
181
|
|
|
76
|
-
function cloneConfigTree(value) {
|
|
77
|
-
if (typeof structuredClone === 'function') {
|
|
78
|
-
return structuredClone(value);
|
|
79
|
-
}
|
|
80
|
-
return JSON.parse(JSON.stringify(value));
|
|
81
|
-
}
|
|
82
|
-
|
|
83
182
|
// =============================================================================
|
|
84
183
|
// Factory Function
|
|
85
184
|
// =============================================================================
|
|
@@ -159,6 +258,7 @@ function mergeSharedRuntimeConfig(
|
|
|
159
258
|
bucket: { ...base.bufferPool.bucket, ...overrides.bufferPool.bucket },
|
|
160
259
|
limits: { ...base.bufferPool.limits, ...overrides.bufferPool.limits },
|
|
161
260
|
alignment: { ...base.bufferPool.alignment, ...overrides.bufferPool.alignment },
|
|
261
|
+
budget: { ...base.bufferPool.budget, ...overrides.bufferPool.budget },
|
|
162
262
|
}
|
|
163
263
|
: { ...base.bufferPool },
|
|
164
264
|
gpuCache: { ...base.gpuCache, ...overrides.gpuCache },
|
|
@@ -241,6 +341,7 @@ function mergeInferenceConfig(
|
|
|
241
341
|
const overrideKernelPathPolicy = overrides.kernelPathPolicy ?? {};
|
|
242
342
|
const baseDiffusion = base.diffusion ?? {};
|
|
243
343
|
const baseDiffusionDecode = baseDiffusion.decode ?? {};
|
|
344
|
+
const baseDiffusionGemma = base.diffusionGemma ?? {};
|
|
244
345
|
const baseEnergy = base.energy ?? {};
|
|
245
346
|
const baseEnergyQuintel = baseEnergy.quintel ?? {};
|
|
246
347
|
const baseMoe = base.moe ?? {};
|
|
@@ -274,6 +375,7 @@ function mergeInferenceConfig(
|
|
|
274
375
|
quantization: { ...baseDiffusion.quantization, ...overrides.diffusion.quantization },
|
|
275
376
|
}
|
|
276
377
|
: { ...baseDiffusion },
|
|
378
|
+
diffusionGemma: { ...baseDiffusionGemma, ...overrides.diffusionGemma },
|
|
277
379
|
energy: overrides.energy
|
|
278
380
|
? {
|
|
279
381
|
...baseEnergy,
|
|
@@ -304,7 +406,7 @@ function mergeInferenceConfig(
|
|
|
304
406
|
speculative: { ...base.speculative, ...overrides.speculative },
|
|
305
407
|
generation: { ...base.generation, ...overrides.generation },
|
|
306
408
|
pipeline: overrides.pipeline ?? base.pipeline,
|
|
307
|
-
kernelPath:
|
|
409
|
+
kernelPath: mergeRuntimeKernelPath(base.kernelPath, overrides.kernelPath),
|
|
308
410
|
kernelPathSource: overrides.kernelPathSource ?? base.kernelPathSource,
|
|
309
411
|
kernelPathPolicy: mergeKernelPathPolicy(baseKernelPathPolicy, overrideKernelPathPolicy),
|
|
310
412
|
chatTemplate: mergeShallowObject(base.chatTemplate, overrides.chatTemplate),
|
|
@@ -325,6 +427,7 @@ function mergeInferenceConfig(
|
|
|
325
427
|
kvcache: replaceSubtree(overrideSession.kvcache, baseSession.kvcache),
|
|
326
428
|
decodeLoop: replaceSubtree(overrideSession.decodeLoop, baseSession.decodeLoop),
|
|
327
429
|
perLayerInputs: replaceSubtree(overrideSession.perLayerInputs, baseSession.perLayerInputs),
|
|
430
|
+
prefillTokenChunkSize: overrideSession.prefillTokenChunkSize ?? baseSession.prefillTokenChunkSize,
|
|
328
431
|
},
|
|
329
432
|
executionPatch: mergeExecutionPatchLists(baseExecutionPatch, overrideExecutionPatch),
|
|
330
433
|
// Model-specific inference overrides (merged with manifest.inference at load time)
|
|
@@ -332,6 +435,32 @@ function mergeInferenceConfig(
|
|
|
332
435
|
};
|
|
333
436
|
}
|
|
334
437
|
|
|
438
|
+
function mergeRuntimeKernelPath(
|
|
439
|
+
baseKernelPath,
|
|
440
|
+
overrideKernelPath
|
|
441
|
+
) {
|
|
442
|
+
const kernelPath = chooseDefined(overrideKernelPath, baseKernelPath);
|
|
443
|
+
assertRuntimeKernelPath(kernelPath);
|
|
444
|
+
return kernelPath;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
function assertRuntimeKernelPath(kernelPath) {
|
|
448
|
+
if (kernelPath === undefined || kernelPath === null) {
|
|
449
|
+
return;
|
|
450
|
+
}
|
|
451
|
+
if (typeof kernelPath === 'string') {
|
|
452
|
+
throw new Error(
|
|
453
|
+
'DopplerConfigError: runtime.inference.kernelPath no longer accepts string registry IDs. ' +
|
|
454
|
+
'Use an inline kernel path object generated from execution-v1, or leave kernelPath null.'
|
|
455
|
+
);
|
|
456
|
+
}
|
|
457
|
+
if (typeof kernelPath !== 'object' || Array.isArray(kernelPath)) {
|
|
458
|
+
throw new Error(
|
|
459
|
+
'DopplerConfigError: runtime.inference.kernelPath must be an inline kernel path object or null.'
|
|
460
|
+
);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
|
|
335
464
|
function mergeKernelThresholds(
|
|
336
465
|
base,
|
|
337
466
|
overrides
|
|
@@ -362,6 +491,9 @@ function mergeDebugConfig(
|
|
|
362
491
|
logLevel: { ...base.logLevel, ...overrides.logLevel },
|
|
363
492
|
trace: { ...base.trace, ...overrides.trace },
|
|
364
493
|
pipeline: { ...base.pipeline, ...overrides.pipeline },
|
|
494
|
+
loader: { ...base.loader, ...overrides.loader },
|
|
495
|
+
matmul: { ...base.matmul, ...overrides.matmul },
|
|
496
|
+
kernelTrace: { ...base.kernelTrace, ...overrides.kernelTrace },
|
|
365
497
|
probes: overrides.probes ?? base.probes,
|
|
366
498
|
profiler: { ...base.profiler, ...overrides.profiler },
|
|
367
499
|
perfGuards: { ...base.perfGuards, ...overrides.perfGuards },
|
|
@@ -349,3 +349,6 @@ export declare function createEmulationConfig(
|
|
|
349
349
|
* Get chip profile config for a specific chip type
|
|
350
350
|
*/
|
|
351
351
|
export declare function getChipProfile(chipType: EmulatedChipType): Partial<EmulationConfigSchema>;
|
|
352
|
+
|
|
353
|
+
/** Format a byte count as a human-readable string (e.g. "1.5 GB"). */
|
|
354
|
+
export declare function formatBytes(bytes: number): string;
|
|
@@ -10,21 +10,21 @@ const gb200Nvl72Profile = await loadJson('../platforms/nvidia-gb200-nvl72.json',
|
|
|
10
10
|
// GPU Specifications
|
|
11
11
|
// =============================================================================
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
const H100_GPU_SPEC = {
|
|
14
14
|
name: 'H100',
|
|
15
15
|
vramBytes: 96 * GB,
|
|
16
16
|
hbmBandwidthBytesPerSec: 3.35e12, // 3.35 TB/s HBM3
|
|
17
17
|
fp16Tflops: 1979, // ~2 PFLOPS
|
|
18
18
|
};
|
|
19
19
|
|
|
20
|
-
|
|
20
|
+
const H200_GPU_SPEC = {
|
|
21
21
|
name: 'H200',
|
|
22
22
|
vramBytes: 144 * GB,
|
|
23
23
|
hbmBandwidthBytesPerSec: 4.8e12, // 4.8 TB/s HBM3e
|
|
24
24
|
fp16Tflops: 1979, // ~2 PFLOPS
|
|
25
25
|
};
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
const B200_GPU_SPEC = {
|
|
28
28
|
name: 'B200',
|
|
29
29
|
vramBytes: 192 * GB,
|
|
30
30
|
hbmBandwidthBytesPerSec: 8e12, // 8 TB/s HBM3e
|
|
@@ -38,7 +38,7 @@ export const DEFAULT_GH200_GPU_SPEC = H200_GPU_SPEC;
|
|
|
38
38
|
// CPU Specifications
|
|
39
39
|
// =============================================================================
|
|
40
40
|
|
|
41
|
-
|
|
41
|
+
const GRACE_CPU_SPEC = {
|
|
42
42
|
name: 'Grace',
|
|
43
43
|
cores: 72,
|
|
44
44
|
memoryBytes: 480 * GB, // LPDDR5X
|
|
@@ -51,12 +51,12 @@ export const DEFAULT_GH200_CPU_SPEC = GRACE_CPU_SPEC;
|
|
|
51
51
|
// NVLink Specifications
|
|
52
52
|
// =============================================================================
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
const NVLINK_4_SPEC = {
|
|
55
55
|
bandwidthBytesPerSec: 900e9, // 900 GB/s
|
|
56
56
|
latencyUs: 1.0, // ~1 microsecond
|
|
57
57
|
};
|
|
58
58
|
|
|
59
|
-
|
|
59
|
+
const NVLINK_5_SPEC = {
|
|
60
60
|
bandwidthBytesPerSec: 1.8e12, // 1.8 TB/s
|
|
61
61
|
latencyUs: 0.8, // ~0.8 microseconds
|
|
62
62
|
};
|
|
@@ -73,28 +73,28 @@ export const DEFAULT_NVLINK_C2C_SPEC = {
|
|
|
73
73
|
// Cluster Topologies
|
|
74
74
|
// =============================================================================
|
|
75
75
|
|
|
76
|
-
|
|
76
|
+
const GH200_TOPOLOGY = {
|
|
77
77
|
gpuCount: 1,
|
|
78
78
|
gpusPerNode: 1,
|
|
79
79
|
nodeCount: 1,
|
|
80
80
|
cpuCount: 1,
|
|
81
81
|
};
|
|
82
82
|
|
|
83
|
-
|
|
83
|
+
const GH200_NVL2_TOPOLOGY = {
|
|
84
84
|
gpuCount: 2,
|
|
85
85
|
gpusPerNode: 2,
|
|
86
86
|
nodeCount: 1,
|
|
87
87
|
cpuCount: 2,
|
|
88
88
|
};
|
|
89
89
|
|
|
90
|
-
|
|
90
|
+
const GB200_8GPU_TOPOLOGY = {
|
|
91
91
|
gpuCount: 8,
|
|
92
92
|
gpusPerNode: 8,
|
|
93
93
|
nodeCount: 1,
|
|
94
94
|
cpuCount: 2,
|
|
95
95
|
};
|
|
96
96
|
|
|
97
|
-
|
|
97
|
+
const GB200_NVL72_TOPOLOGY = {
|
|
98
98
|
gpuCount: 72,
|
|
99
99
|
gpusPerNode: 8,
|
|
100
100
|
nodeCount: 9,
|
|
@@ -125,7 +125,7 @@ export const DEFAULT_PARALLELISM_CONFIG = {
|
|
|
125
125
|
},
|
|
126
126
|
};
|
|
127
127
|
|
|
128
|
-
|
|
128
|
+
const TP2_PARALLELISM_CONFIG = {
|
|
129
129
|
...DEFAULT_PARALLELISM_CONFIG,
|
|
130
130
|
tensorParallel: {
|
|
131
131
|
enabled: true,
|
|
@@ -133,7 +133,7 @@ export const TP2_PARALLELISM_CONFIG = {
|
|
|
133
133
|
},
|
|
134
134
|
};
|
|
135
135
|
|
|
136
|
-
|
|
136
|
+
const TP8_PARALLELISM_CONFIG = {
|
|
137
137
|
...DEFAULT_PARALLELISM_CONFIG,
|
|
138
138
|
tensorParallel: {
|
|
139
139
|
enabled: true,
|
|
@@ -145,7 +145,7 @@ export const TP8_PARALLELISM_CONFIG = {
|
|
|
145
145
|
// Timing Scaling Defaults
|
|
146
146
|
// =============================================================================
|
|
147
147
|
|
|
148
|
-
|
|
148
|
+
const DEFAULT_TIMING_SCALING = {
|
|
149
149
|
computeScale: 1.0,
|
|
150
150
|
memoryScale: 1.0,
|
|
151
151
|
nvlinkScale: 1.0,
|
|
@@ -273,11 +273,11 @@ function mergeParallelismConfig(base, overrides) {
|
|
|
273
273
|
// Utility Functions
|
|
274
274
|
// =============================================================================
|
|
275
275
|
|
|
276
|
-
|
|
276
|
+
function calculateTotalVram(config) {
|
|
277
277
|
return config.gpuSpec.vramBytes * config.topology.gpuCount;
|
|
278
278
|
}
|
|
279
279
|
|
|
280
|
-
|
|
280
|
+
function calculateTotalCpuMemory(config) {
|
|
281
281
|
return config.cpuSpec.memoryBytes * config.topology.cpuCount;
|
|
282
282
|
}
|
|
283
283
|
|
|
@@ -296,6 +296,6 @@ export function formatBytes(bytes) {
|
|
|
296
296
|
return `${bytes} B`;
|
|
297
297
|
}
|
|
298
298
|
|
|
299
|
-
|
|
299
|
+
function formatBandwidth(bytesPerSec) {
|
|
300
300
|
return `${formatBytes(bytesPerSec)}/s`;
|
|
301
301
|
}
|
|
@@ -173,6 +173,30 @@ export interface ExecutionV1SessionSchema {
|
|
|
173
173
|
decodeLoop: ExecutionV1DecodeLoopSchema | null;
|
|
174
174
|
perLayerInputs: ExecutionV1PerLayerInputsSessionSchema;
|
|
175
175
|
speculation: ExecutionV1SelfSpeculationSchema | null;
|
|
176
|
+
/** "sync" (default): wait each prefill chunk. "async": queue without waiting. */
|
|
177
|
+
prefillChunkSubmitMode: 'sync' | 'async';
|
|
178
|
+
/** Nullable token count for streaming prompt prefill before final logits. */
|
|
179
|
+
prefillTokenChunkSize: number | null;
|
|
180
|
+
/** Opt into flash-attention prefill kernel. Requires head_dim=256, f16 KV, contiguous layout. */
|
|
181
|
+
useFlashPrefillAttention: boolean;
|
|
182
|
+
/** Opt into WideTile Q4_K prefill matmul. Requires f32 activations + Q4_K weights + shader-f16 + M>=TILE_M. */
|
|
183
|
+
useWideTileQ4KPrefill: boolean;
|
|
184
|
+
/** Opt into WideTile Q4_K decode matmul. Requires f32 activations + Q4_K weights + shader-f16 + M=1. */
|
|
185
|
+
useWideTileQ4KDecode: boolean;
|
|
186
|
+
/** Opt into two-output sandwich RMSNorm decode fusion. Requires f32 activations + post/pre FFN sandwich norm weights + M=1. */
|
|
187
|
+
useSandwichRMSNormPairFusion: boolean;
|
|
188
|
+
/** Opt into post-FFN plus next-layer input RMSNorm decode fusion. Requires f32 activations + M=1. */
|
|
189
|
+
usePostFfnNextInputRMSNormPairFusion: boolean;
|
|
190
|
+
/** Opt into fused packed-QKV split plus weighted Q/K RMSNorm. Requires f32 QKV output and weighted Q/K norm. */
|
|
191
|
+
useFusedQKVSplitQKNorm: boolean;
|
|
192
|
+
/** Opt into fused packed-QKV split plus weighted Q/K RMSNorm and full-head non-interleaved RoPE. */
|
|
193
|
+
useFusedQKVSplitQKNormRoPE: boolean;
|
|
194
|
+
/** Retain Q4_K packed weights alongside dense buffer (mixed materialization). ~50% extra Q4_K memory. */
|
|
195
|
+
retainQ4KMaterialization: boolean;
|
|
196
|
+
/** Use f32-accumulator twin of fused-Q4K f16a multicol gemv. Closes the f16-accum decode gap on AMD RDNA3 where f32 accumulation outperforms f16 by ~60% at FFN gemv shape. Gated by capability rule. */
|
|
197
|
+
useF32AccumF16ioMatmul: boolean;
|
|
198
|
+
/** Fuse greedy decode LM-head f16-weight GEMV with top-1 selection. Requires greedy sampling with no repetition penalty. */
|
|
199
|
+
useGreedyLmHeadArgmaxFusion: boolean;
|
|
176
200
|
}
|
|
177
201
|
|
|
178
202
|
// === Policies ===
|
|
@@ -232,6 +256,8 @@ export interface ExecutionV1ConfigSchema {
|
|
|
232
256
|
export interface ExecutionV1PatchSetSchema {
|
|
233
257
|
/** Op name to target (matches step[0]) */
|
|
234
258
|
op: string;
|
|
259
|
+
/** Optional graph section to target */
|
|
260
|
+
section?: 'decode' | 'prefill' | 'preLayer' | 'postLayer' | null;
|
|
235
261
|
/** Replace kernel key */
|
|
236
262
|
kernelKey?: string;
|
|
237
263
|
/** Replace weights */
|
|
@@ -243,6 +269,8 @@ export interface ExecutionV1PatchSetSchema {
|
|
|
243
269
|
export interface ExecutionV1PatchRemoveSchema {
|
|
244
270
|
/** Op name to remove */
|
|
245
271
|
op: string;
|
|
272
|
+
/** Optional graph section to target */
|
|
273
|
+
section?: 'decode' | 'prefill' | 'preLayer' | 'postLayer' | null;
|
|
246
274
|
/** Target specific layers only (null = all matching ops) */
|
|
247
275
|
layers?: number[] | null;
|
|
248
276
|
}
|
|
@@ -250,6 +278,8 @@ export interface ExecutionV1PatchRemoveSchema {
|
|
|
250
278
|
export interface ExecutionV1PatchAddSchema {
|
|
251
279
|
/** Step to insert */
|
|
252
280
|
step: ExecutionV1StepTuple;
|
|
281
|
+
/** Optional graph section to target */
|
|
282
|
+
section?: 'decode' | 'prefill' | 'preLayer' | 'postLayer' | null;
|
|
253
283
|
/** Insert before this op */
|
|
254
284
|
insertBefore?: string;
|
|
255
285
|
/** Insert after this op */
|
|
@@ -281,11 +311,16 @@ export interface ExecutionV1PatchSchema {
|
|
|
281
311
|
/** Expanded step — what the runtime actually works with after tuple expansion. */
|
|
282
312
|
export interface ExecutionV1ExpandedStepSchema {
|
|
283
313
|
op: string;
|
|
314
|
+
src: string;
|
|
315
|
+
dst: string;
|
|
284
316
|
kernel: string;
|
|
285
317
|
entry: string;
|
|
286
318
|
digest: string;
|
|
287
319
|
weights: string | null;
|
|
288
320
|
constants: Record<string, number | boolean> | null;
|
|
321
|
+
precision?: ExecutionV1PrecisionSchema;
|
|
322
|
+
fromDtype?: ExecutionV1Dtype;
|
|
323
|
+
toDtype?: ExecutionV1Dtype;
|
|
289
324
|
layers: 'all' | number[];
|
|
290
325
|
phase: 'decode' | 'prefill' | 'both';
|
|
291
326
|
section: 'preLayer' | 'layer' | 'postLayer';
|
|
@@ -295,12 +330,13 @@ export interface ExecutionV1ExpandedStepSchema {
|
|
|
295
330
|
|
|
296
331
|
export declare const EXECUTION_V1_SCHEMA_ID: string;
|
|
297
332
|
export declare const READBACK_MODES: readonly ['sequential', 'overlapped', 'auto'];
|
|
333
|
+
export declare const PREFILL_CHUNK_SUBMIT_MODES: readonly ['sync', 'async'];
|
|
298
334
|
export declare const PER_LAYER_INPUT_MATERIALIZATION_MODES: readonly ['auto', 'range_backed', 'cpu_resident', 'gpu_resident'];
|
|
299
335
|
export declare const PER_LAYER_INPUT_ROW_CACHE_MODES: readonly ['off', 'lru'];
|
|
300
336
|
export declare const PER_LAYER_INPUT_PREFETCH_MODES: readonly ['off', 'next_token'];
|
|
301
337
|
export declare const PER_LAYER_INPUT_GPU_UPLOAD_MODES: readonly ['per_step_slices', 'per_batch_slices'];
|
|
338
|
+
export declare const PER_LAYER_INPUT_HOT_CACHE_MODES: readonly ['off', 'prepared_tokens', 'tokenizer_scores'];
|
|
302
339
|
export declare const DEFAULT_EXECUTION_V1_COMPUTE_DEFAULTS: ExecutionV1ComputeDefaultsSchema;
|
|
303
|
-
export declare const DEFAULT_EXECUTION_V1_PER_LAYER_INPUTS_SESSION: ExecutionV1PerLayerInputsSessionSchema;
|
|
304
340
|
export declare const DEFAULT_EXECUTION_V1_SESSION: ExecutionV1SessionSchema;
|
|
305
341
|
export declare const DEFAULT_EXECUTION_V1_POLICIES: ExecutionV1PoliciesSchema;
|
|
306
342
|
export declare const DEFAULT_EXECUTION_V1_PATCH: ExecutionV1PatchSchema;
|
|
@@ -317,6 +353,8 @@ export interface ExpandExecutionV1Options {
|
|
|
317
353
|
knownOps?: ReadonlySet<string> | null;
|
|
318
354
|
/** When true, unknown ops throw instead of warn. Default false. */
|
|
319
355
|
strict?: boolean;
|
|
356
|
+
/** Skip kernel digest validation for derived transform kernels. */
|
|
357
|
+
skipDigestValidation?: boolean;
|
|
320
358
|
}
|
|
321
359
|
|
|
322
360
|
/** Validate and expand a v1 execution graph into runtime-ready expanded steps. */
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { log } from '../../debug/index.js';
|
|
2
|
+
|
|
1
3
|
// =============================================================================
|
|
2
4
|
// Execution v1 Schema
|
|
3
5
|
// =============================================================================
|
|
@@ -15,6 +17,7 @@ export const DEFAULT_EXECUTION_V1_COMPUTE_DEFAULTS = {
|
|
|
15
17
|
};
|
|
16
18
|
|
|
17
19
|
export const READBACK_MODES = Object.freeze(['sequential', 'overlapped', 'auto']);
|
|
20
|
+
export const PREFILL_CHUNK_SUBMIT_MODES = Object.freeze(['sync', 'async']);
|
|
18
21
|
export const PER_LAYER_INPUT_MATERIALIZATION_MODES = Object.freeze([
|
|
19
22
|
'auto',
|
|
20
23
|
'range_backed',
|
|
@@ -34,7 +37,7 @@ export const PER_LAYER_INPUT_HOT_CACHE_MODES = Object.freeze([
|
|
|
34
37
|
'tokenizer_scores',
|
|
35
38
|
]);
|
|
36
39
|
|
|
37
|
-
|
|
40
|
+
const DEFAULT_EXECUTION_V1_PER_LAYER_INPUTS_SESSION = {
|
|
38
41
|
materialization: 'auto',
|
|
39
42
|
rowCache: {
|
|
40
43
|
mode: 'lru',
|
|
@@ -66,6 +69,18 @@ export const DEFAULT_EXECUTION_V1_SESSION = {
|
|
|
66
69
|
decodeLoop: null,
|
|
67
70
|
perLayerInputs: { ...DEFAULT_EXECUTION_V1_PER_LAYER_INPUTS_SESSION },
|
|
68
71
|
speculation: null,
|
|
72
|
+
prefillChunkSubmitMode: 'sync',
|
|
73
|
+
prefillTokenChunkSize: null,
|
|
74
|
+
useFlashPrefillAttention: false,
|
|
75
|
+
useWideTileQ4KPrefill: false,
|
|
76
|
+
useWideTileQ4KDecode: false,
|
|
77
|
+
useSandwichRMSNormPairFusion: false,
|
|
78
|
+
usePostFfnNextInputRMSNormPairFusion: false,
|
|
79
|
+
useFusedQKVSplitQKNorm: false,
|
|
80
|
+
useFusedQKVSplitQKNormRoPE: false,
|
|
81
|
+
retainQ4KMaterialization: false,
|
|
82
|
+
useF32AccumF16ioMatmul: false,
|
|
83
|
+
useGreedyLmHeadArgmaxFusion: false,
|
|
69
84
|
};
|
|
70
85
|
|
|
71
86
|
export const DEFAULT_EXECUTION_V1_POLICIES = {
|
|
@@ -75,6 +90,7 @@ export const DEFAULT_EXECUTION_V1_POLICIES = {
|
|
|
75
90
|
};
|
|
76
91
|
|
|
77
92
|
export const DEFAULT_EXECUTION_V1_PATCH = {
|
|
93
|
+
addKernels: [],
|
|
78
94
|
set: [],
|
|
79
95
|
remove: [],
|
|
80
96
|
add: [],
|
|
@@ -153,21 +169,34 @@ function expandTuple(tuple, kernels, phase, section, layers, context) {
|
|
|
153
169
|
throw new Error(`${context}: step weights must be a string if provided.`);
|
|
154
170
|
}
|
|
155
171
|
const decl = resolveKernel(kernels, kernelKey, `${context}[${op}]`);
|
|
172
|
+
const precision = decl.precision ?? null;
|
|
173
|
+
const castDtypes = {};
|
|
174
|
+
if (op === 'cast') {
|
|
175
|
+
if (!precision?.inputDtype || !precision?.outputDtype) {
|
|
176
|
+
throw new Error(
|
|
177
|
+
`${context}[cast]: cast steps require kernel precision.inputDtype and precision.outputDtype.`
|
|
178
|
+
);
|
|
179
|
+
}
|
|
180
|
+
castDtypes.fromDtype = precision.inputDtype;
|
|
181
|
+
castDtypes.toDtype = precision.outputDtype;
|
|
182
|
+
}
|
|
156
183
|
return {
|
|
157
184
|
op,
|
|
185
|
+
src: 'state',
|
|
186
|
+
dst: 'state',
|
|
158
187
|
kernel: decl.kernel,
|
|
159
188
|
entry: decl.entry,
|
|
160
189
|
digest: decl.digest,
|
|
161
190
|
weights: weights ?? null,
|
|
162
191
|
constants: decl.constants ?? null,
|
|
163
|
-
...(
|
|
192
|
+
...(precision ? { precision } : {}),
|
|
193
|
+
...castDtypes,
|
|
164
194
|
layers,
|
|
165
195
|
phase,
|
|
166
196
|
section,
|
|
167
197
|
};
|
|
168
198
|
}
|
|
169
199
|
|
|
170
|
-
|
|
171
200
|
function expandStepEntries(entries, kernels, phase, context) {
|
|
172
201
|
const expanded = [];
|
|
173
202
|
for (let i = 0; i < entries.length; i++) {
|
|
@@ -200,16 +229,6 @@ function expandBoundarySteps(entries, kernels, section, context) {
|
|
|
200
229
|
}
|
|
201
230
|
|
|
202
231
|
|
|
203
|
-
/**
|
|
204
|
-
* Expand a v1 execution graph into runtime-ready expanded steps.
|
|
205
|
-
*
|
|
206
|
-
* @param {object} graph - The execution graph from the manifest.
|
|
207
|
-
* @param {{ knownOps?: Set<string> | null, strict?: boolean }} [options]
|
|
208
|
-
* knownOps: when provided, any op not in this set triggers a warning (or
|
|
209
|
-
* throw in strict mode) with the op name and step index.
|
|
210
|
-
* strict: when true, unknown ops throw instead of warn. Default false.
|
|
211
|
-
* @returns {object[]} Expanded step array.
|
|
212
|
-
*/
|
|
213
232
|
export function expandExecutionV1(graph, options = {}) {
|
|
214
233
|
const { knownOps = null, strict = false, skipDigestValidation = false } = options;
|
|
215
234
|
if (!graph || typeof graph !== 'object') {
|
|
@@ -232,9 +251,8 @@ export function expandExecutionV1(graph, options = {}) {
|
|
|
232
251
|
const sections = { preLayer, decode, prefill, postLayer };
|
|
233
252
|
for (const [name, steps] of Object.entries(sections)) {
|
|
234
253
|
if (steps.length > MAX_STEPS_PER_SECTION) {
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
console.warn(
|
|
254
|
+
log.warn(
|
|
255
|
+
'ExecutionV1',
|
|
238
256
|
`[ExecutionV1] Section "${name}" has ${steps.length} expanded steps ` +
|
|
239
257
|
`(max recommended: ${MAX_STEPS_PER_SECTION}). This may indicate a misconfigured execution graph.`
|
|
240
258
|
);
|
|
@@ -261,10 +279,7 @@ export function expandExecutionV1(graph, options = {}) {
|
|
|
261
279
|
if (strict) {
|
|
262
280
|
throw new Error(message);
|
|
263
281
|
}
|
|
264
|
-
|
|
265
|
-
// low-level config layer that should not depend on the debug module.
|
|
266
|
-
// Callers in runtime code can catch the strict error or filter post-hoc.
|
|
267
|
-
console.warn(message);
|
|
282
|
+
log.warn('ExecutionV1', message);
|
|
268
283
|
}
|
|
269
284
|
}
|
|
270
285
|
|