doppler-gpu 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +67 -34
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.js +5 -1
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/version.js +1 -1
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://clocksmith.dev/doppler/schemas/claim-evidence-contract-v1.schema.json",
|
|
4
|
+
"title": "Doppler Claim Evidence Contract v1",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": [
|
|
8
|
+
"$schema",
|
|
9
|
+
"schemaVersion",
|
|
10
|
+
"source",
|
|
11
|
+
"releaseClaimPolicyPath",
|
|
12
|
+
"benchmarkPolicyPath",
|
|
13
|
+
"localInferenceClaimMatrixPath",
|
|
14
|
+
"releaseMatrixPath",
|
|
15
|
+
"goalMatrixPath",
|
|
16
|
+
"requiredScripts",
|
|
17
|
+
"requiredGoalEvidence",
|
|
18
|
+
"requiredReleaseMatrixSources"
|
|
19
|
+
],
|
|
20
|
+
"properties": {
|
|
21
|
+
"$schema": {
|
|
22
|
+
"type": "string",
|
|
23
|
+
"minLength": 1
|
|
24
|
+
},
|
|
25
|
+
"schemaVersion": {
|
|
26
|
+
"const": 1
|
|
27
|
+
},
|
|
28
|
+
"source": {
|
|
29
|
+
"const": "doppler"
|
|
30
|
+
},
|
|
31
|
+
"releaseClaimPolicyPath": {
|
|
32
|
+
"$ref": "#/$defs/repoRelativeJsonPath"
|
|
33
|
+
},
|
|
34
|
+
"benchmarkPolicyPath": {
|
|
35
|
+
"$ref": "#/$defs/repoRelativeJsonPath"
|
|
36
|
+
},
|
|
37
|
+
"localInferenceClaimMatrixPath": {
|
|
38
|
+
"$ref": "#/$defs/repoRelativeJsonPath"
|
|
39
|
+
},
|
|
40
|
+
"releaseMatrixPath": {
|
|
41
|
+
"$ref": "#/$defs/repoRelativeJsonPath"
|
|
42
|
+
},
|
|
43
|
+
"goalMatrixPath": {
|
|
44
|
+
"$ref": "#/$defs/repoRelativeJsonPath"
|
|
45
|
+
},
|
|
46
|
+
"requiredScripts": {
|
|
47
|
+
"type": "array",
|
|
48
|
+
"minItems": 1,
|
|
49
|
+
"items": {
|
|
50
|
+
"type": "string",
|
|
51
|
+
"minLength": 1
|
|
52
|
+
},
|
|
53
|
+
"uniqueItems": true
|
|
54
|
+
},
|
|
55
|
+
"requiredGoalEvidence": {
|
|
56
|
+
"type": "array",
|
|
57
|
+
"minItems": 1,
|
|
58
|
+
"items": {
|
|
59
|
+
"$ref": "#/$defs/repoRelativeJsonPath"
|
|
60
|
+
},
|
|
61
|
+
"uniqueItems": true
|
|
62
|
+
},
|
|
63
|
+
"requiredReleaseMatrixSources": {
|
|
64
|
+
"type": "array",
|
|
65
|
+
"minItems": 1,
|
|
66
|
+
"items": {
|
|
67
|
+
"type": "string",
|
|
68
|
+
"minLength": 1
|
|
69
|
+
},
|
|
70
|
+
"uniqueItems": true
|
|
71
|
+
}
|
|
72
|
+
},
|
|
73
|
+
"$defs": {
|
|
74
|
+
"repoRelativeJsonPath": {
|
|
75
|
+
"type": "string",
|
|
76
|
+
"minLength": 1,
|
|
77
|
+
"pattern": "\\.json$",
|
|
78
|
+
"not": {
|
|
79
|
+
"anyOf": [
|
|
80
|
+
{
|
|
81
|
+
"pattern": "^/"
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
"pattern": "\\\\"
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"pattern": "(^|/)\\.\\.(/|$)"
|
|
88
|
+
}
|
|
89
|
+
]
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://clocksmith.dev/doppler/schemas/command-surface-contract-v1.schema.json",
|
|
4
|
+
"title": "Doppler Command Surface Contract v1",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": [
|
|
8
|
+
"schemaVersion",
|
|
9
|
+
"source",
|
|
10
|
+
"commands"
|
|
11
|
+
],
|
|
12
|
+
"properties": {
|
|
13
|
+
"$schema": {
|
|
14
|
+
"type": "string"
|
|
15
|
+
},
|
|
16
|
+
"schemaVersion": {
|
|
17
|
+
"const": 1
|
|
18
|
+
},
|
|
19
|
+
"source": {
|
|
20
|
+
"const": "doppler"
|
|
21
|
+
},
|
|
22
|
+
"commands": {
|
|
23
|
+
"type": "array",
|
|
24
|
+
"minItems": 1,
|
|
25
|
+
"items": {
|
|
26
|
+
"$ref": "#/$defs/command"
|
|
27
|
+
},
|
|
28
|
+
"uniqueItems": true
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
"$defs": {
|
|
32
|
+
"commandId": {
|
|
33
|
+
"enum": [
|
|
34
|
+
"convert",
|
|
35
|
+
"refresh-integrity",
|
|
36
|
+
"debug",
|
|
37
|
+
"bench",
|
|
38
|
+
"verify",
|
|
39
|
+
"lora",
|
|
40
|
+
"distill",
|
|
41
|
+
"diagnose"
|
|
42
|
+
]
|
|
43
|
+
},
|
|
44
|
+
"surface": {
|
|
45
|
+
"enum": [
|
|
46
|
+
"browser",
|
|
47
|
+
"node"
|
|
48
|
+
]
|
|
49
|
+
},
|
|
50
|
+
"command": {
|
|
51
|
+
"type": "object",
|
|
52
|
+
"additionalProperties": false,
|
|
53
|
+
"required": [
|
|
54
|
+
"id",
|
|
55
|
+
"surfaces",
|
|
56
|
+
"cliUsageRequired",
|
|
57
|
+
"reason"
|
|
58
|
+
],
|
|
59
|
+
"properties": {
|
|
60
|
+
"id": {
|
|
61
|
+
"$ref": "#/$defs/commandId"
|
|
62
|
+
},
|
|
63
|
+
"surfaces": {
|
|
64
|
+
"type": "array",
|
|
65
|
+
"minItems": 1,
|
|
66
|
+
"items": {
|
|
67
|
+
"$ref": "#/$defs/surface"
|
|
68
|
+
},
|
|
69
|
+
"uniqueItems": true
|
|
70
|
+
},
|
|
71
|
+
"cliUsageRequired": {
|
|
72
|
+
"type": "boolean"
|
|
73
|
+
},
|
|
74
|
+
"reason": {
|
|
75
|
+
"type": "string",
|
|
76
|
+
"minLength": 1
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
@@ -9,10 +9,14 @@
|
|
|
9
9
|
import type {
|
|
10
10
|
HashAlgorithm,
|
|
11
11
|
QuantizationValue,
|
|
12
|
+
SourceQuantizationFormat,
|
|
13
|
+
SourceQuantizationTarget,
|
|
14
|
+
SourceTrainingQuantization,
|
|
12
15
|
RuntimeOptimizationsSchema,
|
|
13
16
|
ConversionInfoSchema,
|
|
14
17
|
ManifestArtifactIdentitySchema,
|
|
15
18
|
ManifestWeightsRefSchema,
|
|
19
|
+
MoEConfigSchema,
|
|
16
20
|
} from './manifest.schema.js';
|
|
17
21
|
import type {
|
|
18
22
|
ExecutionV1GraphSchema,
|
|
@@ -28,6 +32,9 @@ export interface ConverterQuantizationConfigSchema {
|
|
|
28
32
|
weights: QuantizationValue | null;
|
|
29
33
|
embeddings: QuantizationValue | null;
|
|
30
34
|
lmHead: QuantizationValue | null;
|
|
35
|
+
sourceTrainingQuantization: SourceTrainingQuantization | null;
|
|
36
|
+
sourceQuantizationTarget: SourceQuantizationTarget | null;
|
|
37
|
+
sourceQuantizationFormat: SourceQuantizationFormat | null;
|
|
31
38
|
vision: QuantizationValue | null;
|
|
32
39
|
audio: QuantizationValue | null;
|
|
33
40
|
projector: QuantizationValue | null;
|
|
@@ -105,6 +112,10 @@ export interface ConverterConfigSchema {
|
|
|
105
112
|
manifest: ConverterManifestConfigSchema;
|
|
106
113
|
inference: ConverterInferenceConfigSchema;
|
|
107
114
|
output: ConverterOutputConfigSchema;
|
|
115
|
+
moeConfig?: MoEConfigSchema | null;
|
|
116
|
+
execution?: ExecutionV1GraphSchema;
|
|
117
|
+
session?: ExecutionV1SessionSchema;
|
|
118
|
+
modelType?: string;
|
|
108
119
|
}
|
|
109
120
|
|
|
110
121
|
export declare const DEFAULT_CONVERTER_QUANTIZATION_CONFIG: ConverterQuantizationConfigSchema;
|
|
@@ -10,6 +10,9 @@ export const DEFAULT_CONVERTER_QUANTIZATION_CONFIG = {
|
|
|
10
10
|
weights: null,
|
|
11
11
|
embeddings: null,
|
|
12
12
|
lmHead: null,
|
|
13
|
+
sourceTrainingQuantization: null,
|
|
14
|
+
sourceQuantizationTarget: null,
|
|
15
|
+
sourceQuantizationFormat: null,
|
|
13
16
|
vision: DEFAULT_QUANTIZATION_DEFAULTS.visionDtype,
|
|
14
17
|
audio: DEFAULT_QUANTIZATION_DEFAULTS.audioDtype,
|
|
15
18
|
projector: DEFAULT_QUANTIZATION_DEFAULTS.projectorDtype,
|
|
@@ -44,6 +47,7 @@ export const DEFAULT_CONVERTER_MANIFEST_CONFIG = {
|
|
|
44
47
|
conversion: null,
|
|
45
48
|
artifactIdentity: null,
|
|
46
49
|
weightsRef: null,
|
|
50
|
+
eosTokenId: null,
|
|
47
51
|
visionConfig: null,
|
|
48
52
|
audioConfig: null,
|
|
49
53
|
};
|
|
@@ -87,6 +91,7 @@ export const DEFAULT_CONVERTER_CONFIG = {
|
|
|
87
91
|
manifest: DEFAULT_CONVERTER_MANIFEST_CONFIG,
|
|
88
92
|
inference: DEFAULT_CONVERTER_INFERENCE_CONFIG,
|
|
89
93
|
output: DEFAULT_CONVERTER_OUTPUT_CONFIG,
|
|
94
|
+
moeConfig: null,
|
|
90
95
|
};
|
|
91
96
|
|
|
92
97
|
export function createConverterConfig(overrides) {
|
|
@@ -128,6 +133,9 @@ export function createConverterConfig(overrides) {
|
|
|
128
133
|
output: overrides.output
|
|
129
134
|
? { ...DEFAULT_CONVERTER_OUTPUT_CONFIG, ...overrides.output }
|
|
130
135
|
: { ...DEFAULT_CONVERTER_OUTPUT_CONFIG },
|
|
136
|
+
moeConfig: Object.prototype.hasOwnProperty.call(overrides, 'moeConfig')
|
|
137
|
+
? (overrides.moeConfig ?? null)
|
|
138
|
+
: null,
|
|
131
139
|
};
|
|
132
140
|
// V1 conversion configs place execution, session, and modelType at the top level.
|
|
133
141
|
// Pass them through so isV1Config() can detect the v1 format and resolveConversionPlanV1
|
|
@@ -38,6 +38,7 @@ export interface RuntimeInferenceConfigSchema {
|
|
|
38
38
|
largeWeights: LargeWeightConfigSchema;
|
|
39
39
|
kvcache: Record<string, unknown>;
|
|
40
40
|
diffusion: Record<string, unknown>;
|
|
41
|
+
diffusionGemma: Record<string, unknown>;
|
|
41
42
|
energy: Record<string, unknown>;
|
|
42
43
|
moe: Record<string, unknown>;
|
|
43
44
|
speculative: Record<string, unknown>;
|
|
@@ -45,12 +45,16 @@ export const DEFAULT_RUNTIME_CONFIG = {
|
|
|
45
45
|
repetitionPenalty: 1.1,
|
|
46
46
|
greedyThreshold: 0.01,
|
|
47
47
|
repetitionPenaltyWindow: 100,
|
|
48
|
+
suppressSpecialTokens: false,
|
|
49
|
+
suppressSpecialLikeTokens: false,
|
|
50
|
+
suppressTokenIds: [],
|
|
48
51
|
},
|
|
49
52
|
compute: {},
|
|
50
53
|
tokenizer: {},
|
|
51
54
|
largeWeights: DEFAULT_LARGE_WEIGHT_CONFIG,
|
|
52
55
|
kvcache: DEFAULT_KVCACHE_CONFIG,
|
|
53
56
|
diffusion: {},
|
|
57
|
+
diffusionGemma: {},
|
|
54
58
|
energy: {},
|
|
55
59
|
moe: DEFAULT_MOE_RUNTIME_CONFIG,
|
|
56
60
|
speculative: DEFAULT_SPECULATIVE_CONFIG,
|
|
@@ -79,6 +83,7 @@ export const DEFAULT_RUNTIME_CONFIG = {
|
|
|
79
83
|
// transitions but longer intermediate-buffer lifetime. Default 4 is
|
|
80
84
|
// memory-conservative; profiles with known-safe memory can raise it.
|
|
81
85
|
prefillChunkLayers: 4,
|
|
86
|
+
prefillTokenChunkSize: null,
|
|
82
87
|
// Opt into the flash-attention prefill kernel (head_dim=256, f16 KV,
|
|
83
88
|
// contiguous layout). Two-pass dispatch raises RDNA3 workgroup
|
|
84
89
|
// occupancy by KV-axis splitting. The kernel itself enforces head_dim
|
|
@@ -113,6 +118,22 @@ export const DEFAULT_RUNTIME_CONFIG = {
|
|
|
113
118
|
// Default false until correctness + perf parity validated on target
|
|
114
119
|
// hardware.
|
|
115
120
|
useWideTileQ4KPrefill: false,
|
|
121
|
+
// Opt into the WideTile Q4_K decode matmul. This reuses the WideTile
|
|
122
|
+
// register kernel for one-row decode matmuls when explicitly enabled.
|
|
123
|
+
// Default false until correctness + perf parity validated on target
|
|
124
|
+
// hardware.
|
|
125
|
+
useWideTileQ4KDecode: false,
|
|
126
|
+
// Opt into a two-output sandwich RMSNorm decode kernel:
|
|
127
|
+
// post_attn_norm and pre_ffn_norm execute in one dispatch while still
|
|
128
|
+
// materializing both tensors for the downstream residual and FFN paths.
|
|
129
|
+
// Default false until correctness + perf validated.
|
|
130
|
+
useSandwichRMSNormPairFusion: false,
|
|
131
|
+
// Opt into a decode-only cross-layer RMSNorm pair:
|
|
132
|
+
// post_ffn_norm for layer N and input_norm for layer N+1 execute in one
|
|
133
|
+
// dispatch. The next layer consumes the precomputed input-norm tensor at
|
|
134
|
+
// its normal observation point. Default false until correctness + perf
|
|
135
|
+
// validated.
|
|
136
|
+
usePostFfnNextInputRMSNormPairFusion: false,
|
|
116
137
|
// Opt into the single-pass flash attention prefill kernel adapted from
|
|
117
138
|
// ORT's flash_attention.wgsl.template. 64 threads = 64 queries per WG;
|
|
118
139
|
// private Q/O tiles, shared K/V tiles, online softmax, no reduce pass.
|
|
@@ -123,19 +144,26 @@ export const DEFAULT_RUNTIME_CONFIG = {
|
|
|
123
144
|
useOrtFlashPrefillAttention: false,
|
|
124
145
|
// Opt into the WideTile Q4_K matmul + residual fusion at the ffn_down
|
|
125
146
|
// + ffn_residual call site (dense.js). Eliminates one separate residual
|
|
126
|
-
// dispatch per layer
|
|
127
|
-
//
|
|
147
|
+
// dispatch per layer. Requires f32 activations + retained Q4_K weights
|
|
148
|
+
// plus an enabled WideTile phase.
|
|
128
149
|
// Default false until correctness + perf validated.
|
|
129
150
|
useWideTileResidualFusion: false,
|
|
130
151
|
// Opt into the RMSNorm + WideTile Q4_K matmul fusion at pre-matmul
|
|
131
152
|
// norm sites (input_norm→q/k/v_proj, pre_feedforward_norm→gate/up).
|
|
132
153
|
// Each fused call recomputes RMS internally (redundant across q/k/v
|
|
133
154
|
// but negligible) and skips the standalone rmsnorm dispatch upstream.
|
|
134
|
-
//
|
|
135
|
-
//
|
|
136
|
-
// weights retained + prefill. Default false until wiring lands and
|
|
137
|
-
// correctness validates.
|
|
155
|
+
// Requires f32 activations + Q4_K weights retained + prefill. Default
|
|
156
|
+
// false until wiring lands and correctness validates.
|
|
138
157
|
useFusedRmsnormWideTile: false,
|
|
158
|
+
// Opt into fused packed-QKV split plus weighted Q/K RMSNorm. Replaces
|
|
159
|
+
// split_qkv + rmsnorm_qk when diagnostics do not need raw Q/K projection
|
|
160
|
+
// readbacks. Requires f32 QKV output and weighted Q/K norm.
|
|
161
|
+
useFusedQKVSplitQKNorm: false,
|
|
162
|
+
// Opt into fused packed-QKV split plus weighted Q/K RMSNorm and full-head
|
|
163
|
+
// non-interleaved RoPE. Replaces split_qkv + rmsnorm_qk + rope_qk when
|
|
164
|
+
// diagnostics do not need intermediate Q/K stage readbacks.
|
|
165
|
+
useFusedQKVSplitQKNormRoPE: false,
|
|
166
|
+
useGreedyLmHeadArgmaxFusion: false,
|
|
139
167
|
},
|
|
140
168
|
executionPatch: {},
|
|
141
169
|
},
|
|
@@ -230,6 +258,7 @@ function mergeSharedRuntimeConfig(
|
|
|
230
258
|
bucket: { ...base.bufferPool.bucket, ...overrides.bufferPool.bucket },
|
|
231
259
|
limits: { ...base.bufferPool.limits, ...overrides.bufferPool.limits },
|
|
232
260
|
alignment: { ...base.bufferPool.alignment, ...overrides.bufferPool.alignment },
|
|
261
|
+
budget: { ...base.bufferPool.budget, ...overrides.bufferPool.budget },
|
|
233
262
|
}
|
|
234
263
|
: { ...base.bufferPool },
|
|
235
264
|
gpuCache: { ...base.gpuCache, ...overrides.gpuCache },
|
|
@@ -312,6 +341,7 @@ function mergeInferenceConfig(
|
|
|
312
341
|
const overrideKernelPathPolicy = overrides.kernelPathPolicy ?? {};
|
|
313
342
|
const baseDiffusion = base.diffusion ?? {};
|
|
314
343
|
const baseDiffusionDecode = baseDiffusion.decode ?? {};
|
|
344
|
+
const baseDiffusionGemma = base.diffusionGemma ?? {};
|
|
315
345
|
const baseEnergy = base.energy ?? {};
|
|
316
346
|
const baseEnergyQuintel = baseEnergy.quintel ?? {};
|
|
317
347
|
const baseMoe = base.moe ?? {};
|
|
@@ -345,6 +375,7 @@ function mergeInferenceConfig(
|
|
|
345
375
|
quantization: { ...baseDiffusion.quantization, ...overrides.diffusion.quantization },
|
|
346
376
|
}
|
|
347
377
|
: { ...baseDiffusion },
|
|
378
|
+
diffusionGemma: { ...baseDiffusionGemma, ...overrides.diffusionGemma },
|
|
348
379
|
energy: overrides.energy
|
|
349
380
|
? {
|
|
350
381
|
...baseEnergy,
|
|
@@ -396,6 +427,7 @@ function mergeInferenceConfig(
|
|
|
396
427
|
kvcache: replaceSubtree(overrideSession.kvcache, baseSession.kvcache),
|
|
397
428
|
decodeLoop: replaceSubtree(overrideSession.decodeLoop, baseSession.decodeLoop),
|
|
398
429
|
perLayerInputs: replaceSubtree(overrideSession.perLayerInputs, baseSession.perLayerInputs),
|
|
430
|
+
prefillTokenChunkSize: overrideSession.prefillTokenChunkSize ?? baseSession.prefillTokenChunkSize,
|
|
399
431
|
},
|
|
400
432
|
executionPatch: mergeExecutionPatchLists(baseExecutionPatch, overrideExecutionPatch),
|
|
401
433
|
// Model-specific inference overrides (merged with manifest.inference at load time)
|
|
@@ -175,14 +175,28 @@ export interface ExecutionV1SessionSchema {
|
|
|
175
175
|
speculation: ExecutionV1SelfSpeculationSchema | null;
|
|
176
176
|
/** "sync" (default): wait each prefill chunk. "async": queue without waiting. */
|
|
177
177
|
prefillChunkSubmitMode: 'sync' | 'async';
|
|
178
|
+
/** Nullable token count for streaming prompt prefill before final logits. */
|
|
179
|
+
prefillTokenChunkSize: number | null;
|
|
178
180
|
/** Opt into flash-attention prefill kernel. Requires head_dim=256, f16 KV, contiguous layout. */
|
|
179
181
|
useFlashPrefillAttention: boolean;
|
|
180
182
|
/** Opt into WideTile Q4_K prefill matmul. Requires f32 activations + Q4_K weights + shader-f16 + M>=TILE_M. */
|
|
181
183
|
useWideTileQ4KPrefill: boolean;
|
|
184
|
+
/** Opt into WideTile Q4_K decode matmul. Requires f32 activations + Q4_K weights + shader-f16 + M=1. */
|
|
185
|
+
useWideTileQ4KDecode: boolean;
|
|
186
|
+
/** Opt into two-output sandwich RMSNorm decode fusion. Requires f32 activations + post/pre FFN sandwich norm weights + M=1. */
|
|
187
|
+
useSandwichRMSNormPairFusion: boolean;
|
|
188
|
+
/** Opt into post-FFN plus next-layer input RMSNorm decode fusion. Requires f32 activations + M=1. */
|
|
189
|
+
usePostFfnNextInputRMSNormPairFusion: boolean;
|
|
190
|
+
/** Opt into fused packed-QKV split plus weighted Q/K RMSNorm. Requires f32 QKV output and weighted Q/K norm. */
|
|
191
|
+
useFusedQKVSplitQKNorm: boolean;
|
|
192
|
+
/** Opt into fused packed-QKV split plus weighted Q/K RMSNorm and full-head non-interleaved RoPE. */
|
|
193
|
+
useFusedQKVSplitQKNormRoPE: boolean;
|
|
182
194
|
/** Retain Q4_K packed weights alongside dense buffer (mixed materialization). ~50% extra Q4_K memory. */
|
|
183
195
|
retainQ4KMaterialization: boolean;
|
|
184
196
|
/** Use f32-accumulator twin of fused-Q4K f16a multicol gemv. Closes the f16-accum decode gap on AMD RDNA3 where f32 accumulation outperforms f16 by ~60% at FFN gemv shape. Gated by capability rule. */
|
|
185
197
|
useF32AccumF16ioMatmul: boolean;
|
|
198
|
+
/** Fuse greedy decode LM-head f16-weight GEMV with top-1 selection. Requires greedy sampling with no repetition penalty. */
|
|
199
|
+
useGreedyLmHeadArgmaxFusion: boolean;
|
|
186
200
|
}
|
|
187
201
|
|
|
188
202
|
// === Policies ===
|
|
@@ -242,6 +256,8 @@ export interface ExecutionV1ConfigSchema {
|
|
|
242
256
|
export interface ExecutionV1PatchSetSchema {
|
|
243
257
|
/** Op name to target (matches step[0]) */
|
|
244
258
|
op: string;
|
|
259
|
+
/** Optional graph section to target */
|
|
260
|
+
section?: 'decode' | 'prefill' | 'preLayer' | 'postLayer' | null;
|
|
245
261
|
/** Replace kernel key */
|
|
246
262
|
kernelKey?: string;
|
|
247
263
|
/** Replace weights */
|
|
@@ -253,6 +269,8 @@ export interface ExecutionV1PatchSetSchema {
|
|
|
253
269
|
export interface ExecutionV1PatchRemoveSchema {
|
|
254
270
|
/** Op name to remove */
|
|
255
271
|
op: string;
|
|
272
|
+
/** Optional graph section to target */
|
|
273
|
+
section?: 'decode' | 'prefill' | 'preLayer' | 'postLayer' | null;
|
|
256
274
|
/** Target specific layers only (null = all matching ops) */
|
|
257
275
|
layers?: number[] | null;
|
|
258
276
|
}
|
|
@@ -260,6 +278,8 @@ export interface ExecutionV1PatchRemoveSchema {
|
|
|
260
278
|
export interface ExecutionV1PatchAddSchema {
|
|
261
279
|
/** Step to insert */
|
|
262
280
|
step: ExecutionV1StepTuple;
|
|
281
|
+
/** Optional graph section to target */
|
|
282
|
+
section?: 'decode' | 'prefill' | 'preLayer' | 'postLayer' | null;
|
|
263
283
|
/** Insert before this op */
|
|
264
284
|
insertBefore?: string;
|
|
265
285
|
/** Insert after this op */
|
|
@@ -291,11 +311,16 @@ export interface ExecutionV1PatchSchema {
|
|
|
291
311
|
/** Expanded step — what the runtime actually works with after tuple expansion. */
|
|
292
312
|
export interface ExecutionV1ExpandedStepSchema {
|
|
293
313
|
op: string;
|
|
314
|
+
src: string;
|
|
315
|
+
dst: string;
|
|
294
316
|
kernel: string;
|
|
295
317
|
entry: string;
|
|
296
318
|
digest: string;
|
|
297
319
|
weights: string | null;
|
|
298
320
|
constants: Record<string, number | boolean> | null;
|
|
321
|
+
precision?: ExecutionV1PrecisionSchema;
|
|
322
|
+
fromDtype?: ExecutionV1Dtype;
|
|
323
|
+
toDtype?: ExecutionV1Dtype;
|
|
299
324
|
layers: 'all' | number[];
|
|
300
325
|
phase: 'decode' | 'prefill' | 'both';
|
|
301
326
|
section: 'preLayer' | 'layer' | 'postLayer';
|
|
@@ -328,6 +353,8 @@ export interface ExpandExecutionV1Options {
|
|
|
328
353
|
knownOps?: ReadonlySet<string> | null;
|
|
329
354
|
/** When true, unknown ops throw instead of warn. Default false. */
|
|
330
355
|
strict?: boolean;
|
|
356
|
+
/** Skip kernel digest validation for derived transform kernels. */
|
|
357
|
+
skipDigestValidation?: boolean;
|
|
331
358
|
}
|
|
332
359
|
|
|
333
360
|
/** Validate and expand a v1 execution graph into runtime-ready expanded steps. */
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { log } from '../../debug/index.js';
|
|
2
|
+
|
|
1
3
|
// =============================================================================
|
|
2
4
|
// Execution v1 Schema
|
|
3
5
|
// =============================================================================
|
|
@@ -68,10 +70,17 @@ export const DEFAULT_EXECUTION_V1_SESSION = {
|
|
|
68
70
|
perLayerInputs: { ...DEFAULT_EXECUTION_V1_PER_LAYER_INPUTS_SESSION },
|
|
69
71
|
speculation: null,
|
|
70
72
|
prefillChunkSubmitMode: 'sync',
|
|
73
|
+
prefillTokenChunkSize: null,
|
|
71
74
|
useFlashPrefillAttention: false,
|
|
72
75
|
useWideTileQ4KPrefill: false,
|
|
76
|
+
useWideTileQ4KDecode: false,
|
|
77
|
+
useSandwichRMSNormPairFusion: false,
|
|
78
|
+
usePostFfnNextInputRMSNormPairFusion: false,
|
|
79
|
+
useFusedQKVSplitQKNorm: false,
|
|
80
|
+
useFusedQKVSplitQKNormRoPE: false,
|
|
73
81
|
retainQ4KMaterialization: false,
|
|
74
82
|
useF32AccumF16ioMatmul: false,
|
|
83
|
+
useGreedyLmHeadArgmaxFusion: false,
|
|
75
84
|
};
|
|
76
85
|
|
|
77
86
|
export const DEFAULT_EXECUTION_V1_POLICIES = {
|
|
@@ -81,6 +90,7 @@ export const DEFAULT_EXECUTION_V1_POLICIES = {
|
|
|
81
90
|
};
|
|
82
91
|
|
|
83
92
|
export const DEFAULT_EXECUTION_V1_PATCH = {
|
|
93
|
+
addKernels: [],
|
|
84
94
|
set: [],
|
|
85
95
|
remove: [],
|
|
86
96
|
add: [],
|
|
@@ -159,21 +169,34 @@ function expandTuple(tuple, kernels, phase, section, layers, context) {
|
|
|
159
169
|
throw new Error(`${context}: step weights must be a string if provided.`);
|
|
160
170
|
}
|
|
161
171
|
const decl = resolveKernel(kernels, kernelKey, `${context}[${op}]`);
|
|
172
|
+
const precision = decl.precision ?? null;
|
|
173
|
+
const castDtypes = {};
|
|
174
|
+
if (op === 'cast') {
|
|
175
|
+
if (!precision?.inputDtype || !precision?.outputDtype) {
|
|
176
|
+
throw new Error(
|
|
177
|
+
`${context}[cast]: cast steps require kernel precision.inputDtype and precision.outputDtype.`
|
|
178
|
+
);
|
|
179
|
+
}
|
|
180
|
+
castDtypes.fromDtype = precision.inputDtype;
|
|
181
|
+
castDtypes.toDtype = precision.outputDtype;
|
|
182
|
+
}
|
|
162
183
|
return {
|
|
163
184
|
op,
|
|
185
|
+
src: 'state',
|
|
186
|
+
dst: 'state',
|
|
164
187
|
kernel: decl.kernel,
|
|
165
188
|
entry: decl.entry,
|
|
166
189
|
digest: decl.digest,
|
|
167
190
|
weights: weights ?? null,
|
|
168
191
|
constants: decl.constants ?? null,
|
|
169
|
-
...(
|
|
192
|
+
...(precision ? { precision } : {}),
|
|
193
|
+
...castDtypes,
|
|
170
194
|
layers,
|
|
171
195
|
phase,
|
|
172
196
|
section,
|
|
173
197
|
};
|
|
174
198
|
}
|
|
175
199
|
|
|
176
|
-
|
|
177
200
|
function expandStepEntries(entries, kernels, phase, context) {
|
|
178
201
|
const expanded = [];
|
|
179
202
|
for (let i = 0; i < entries.length; i++) {
|
|
@@ -206,16 +229,6 @@ function expandBoundarySteps(entries, kernels, section, context) {
|
|
|
206
229
|
}
|
|
207
230
|
|
|
208
231
|
|
|
209
|
-
/**
|
|
210
|
-
* Expand a v1 execution graph into runtime-ready expanded steps.
|
|
211
|
-
*
|
|
212
|
-
* @param {object} graph - The execution graph from the manifest.
|
|
213
|
-
* @param {{ knownOps?: Set<string> | null, strict?: boolean }} [options]
|
|
214
|
-
* knownOps: when provided, any op not in this set triggers a warning (or
|
|
215
|
-
* throw in strict mode) with the op name and step index.
|
|
216
|
-
* strict: when true, unknown ops throw instead of warn. Default false.
|
|
217
|
-
* @returns {object[]} Expanded step array.
|
|
218
|
-
*/
|
|
219
232
|
export function expandExecutionV1(graph, options = {}) {
|
|
220
233
|
const { knownOps = null, strict = false, skipDigestValidation = false } = options;
|
|
221
234
|
if (!graph || typeof graph !== 'object') {
|
|
@@ -238,9 +251,8 @@ export function expandExecutionV1(graph, options = {}) {
|
|
|
238
251
|
const sections = { preLayer, decode, prefill, postLayer };
|
|
239
252
|
for (const [name, steps] of Object.entries(sections)) {
|
|
240
253
|
if (steps.length > MAX_STEPS_PER_SECTION) {
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
console.warn(
|
|
254
|
+
log.warn(
|
|
255
|
+
'ExecutionV1',
|
|
244
256
|
`[ExecutionV1] Section "${name}" has ${steps.length} expanded steps ` +
|
|
245
257
|
`(max recommended: ${MAX_STEPS_PER_SECTION}). This may indicate a misconfigured execution graph.`
|
|
246
258
|
);
|
|
@@ -267,10 +279,7 @@ export function expandExecutionV1(graph, options = {}) {
|
|
|
267
279
|
if (strict) {
|
|
268
280
|
throw new Error(message);
|
|
269
281
|
}
|
|
270
|
-
|
|
271
|
-
// low-level config layer that should not depend on the debug module.
|
|
272
|
-
// Callers in runtime code can catch the strict error or filter post-hoc.
|
|
273
|
-
console.warn(message);
|
|
282
|
+
log.warn('ExecutionV1', message);
|
|
274
283
|
}
|
|
275
284
|
}
|
|
276
285
|
|