doppler-gpu 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +67 -34
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.js +5 -1
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/version.js +1 -1
|
@@ -7,7 +7,12 @@
|
|
|
7
7
|
* @module inference/pipelines/text/ops
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
import type {
|
|
10
|
+
import type {
|
|
11
|
+
SiLURowSplitOptions,
|
|
12
|
+
CommandRecorder,
|
|
13
|
+
SandwichRMSNormPairOptions,
|
|
14
|
+
SandwichRMSNormPairResult,
|
|
15
|
+
} from '../../../gpu/kernel-selector.js';
|
|
11
16
|
import type { Tensor } from '../../../gpu/tensor.js';
|
|
12
17
|
import type { WeightBuffer, CpuWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
13
18
|
import type { DecodeBufferManager } from '../../decode-buffers.js';
|
|
@@ -35,10 +40,31 @@ export function doRMSNorm(
|
|
|
35
40
|
input: Tensor,
|
|
36
41
|
weight: GPUBuffer,
|
|
37
42
|
eps: number,
|
|
38
|
-
options: {
|
|
43
|
+
options: {
|
|
44
|
+
batchSize: number;
|
|
45
|
+
hiddenSize: number;
|
|
46
|
+
residual?: Tensor | null;
|
|
47
|
+
preResidual?: Tensor | null;
|
|
48
|
+
residualSumOutput?: GPUBuffer | Tensor | null;
|
|
49
|
+
outputBuffer?: GPUBuffer | null;
|
|
50
|
+
outputScale?: number | null;
|
|
51
|
+
label?: string;
|
|
52
|
+
layerIdx?: number;
|
|
53
|
+
rmsNormWeightOffset?: boolean;
|
|
54
|
+
},
|
|
39
55
|
recorder?: CommandRecorder
|
|
40
56
|
): Promise<Tensor>;
|
|
41
57
|
|
|
58
|
+
export function doSandwichRMSNormPair(
|
|
59
|
+
input: Tensor,
|
|
60
|
+
residual: Tensor | null,
|
|
61
|
+
postWeight: GPUBuffer | WeightBuffer | CpuWeightBuffer,
|
|
62
|
+
preWeight: GPUBuffer | WeightBuffer | CpuWeightBuffer,
|
|
63
|
+
eps: number,
|
|
64
|
+
options?: SandwichRMSNormPairOptions,
|
|
65
|
+
recorder?: CommandRecorder
|
|
66
|
+
): Promise<SandwichRMSNormPairResult>;
|
|
67
|
+
|
|
42
68
|
/**
|
|
43
69
|
* ResidualAdd that uses record variant when recorder is provided.
|
|
44
70
|
* Accepts Tensor for inputs, returns Tensor.
|
|
@@ -52,6 +78,7 @@ export function doResidualAdd(
|
|
|
52
78
|
label?: string;
|
|
53
79
|
layerIdx?: number;
|
|
54
80
|
outputBuffer?: GPUBuffer | null;
|
|
81
|
+
outputScale?: number | null;
|
|
55
82
|
executionPolicies?: ExecutionV1PoliciesSchema | null;
|
|
56
83
|
}
|
|
57
84
|
): Promise<Tensor>;
|
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
recordRMSNorm, recordResidualAdd, recordMatmul, recordSiLU, recordGeLU,
|
|
6
6
|
runSiLURowSplit, recordSiLURowSplit,
|
|
7
7
|
runMatmulRMSNormFused, recordMatmulRMSNormFused,
|
|
8
|
+
runSandwichRMSNormPair, recordSandwichRMSNormPair,
|
|
8
9
|
runConv2D, recordConv2D,
|
|
9
10
|
} from '../../../gpu/kernel-selector.js';
|
|
10
11
|
import {
|
|
@@ -57,6 +58,21 @@ export async function doRMSNorm(input, weight, eps, options, recorder) {
|
|
|
57
58
|
return result;
|
|
58
59
|
}
|
|
59
60
|
|
|
61
|
+
export async function doSandwichRMSNormPair(input, residual, postWeight, preWeight, eps, options, recorder) {
|
|
62
|
+
const result = recorder
|
|
63
|
+
? await recordSandwichRMSNormPair(recorder, input, residual, postWeight, preWeight, eps, options)
|
|
64
|
+
: await runSandwichRMSNormPair(input, residual, postWeight, preWeight, eps, options);
|
|
65
|
+
|
|
66
|
+
if (kernelTrace.enabled && !recorder) {
|
|
67
|
+
const layer = options.layerIdx ?? -1;
|
|
68
|
+
const label = options.label ?? 'rmsnorm_pair';
|
|
69
|
+
await traceStep('rmsnorm_pair.post_attn', label, layer, result.postAttn.buffer, [options.batchSize, options.hiddenSize]);
|
|
70
|
+
await traceStep('rmsnorm_pair.pre_ffn', label, layer, result.ffnInput.buffer, [options.batchSize, options.hiddenSize]);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
return result;
|
|
74
|
+
}
|
|
75
|
+
|
|
60
76
|
|
|
61
77
|
export async function doResidualAdd(a, b, size, recorder, traceOptions) {
|
|
62
78
|
const options = {};
|
|
@@ -66,6 +82,9 @@ export async function doResidualAdd(a, b, size, recorder, traceOptions) {
|
|
|
66
82
|
if (traceOptions?.executionPolicies) {
|
|
67
83
|
options.executionPolicies = traceOptions.executionPolicies;
|
|
68
84
|
}
|
|
85
|
+
if (traceOptions?.outputScale !== undefined) {
|
|
86
|
+
options.outputScale = traceOptions.outputScale;
|
|
87
|
+
}
|
|
69
88
|
const result = recorder
|
|
70
89
|
? await recordResidualAdd(recorder, a, b, size, options)
|
|
71
90
|
: await runResidualAdd(a, b, size, options);
|
|
@@ -872,6 +872,11 @@ export function hasGpuSplitPerLayerInputEmbeddings(context) {
|
|
|
872
872
|
return false;
|
|
873
873
|
}
|
|
874
874
|
|
|
875
|
+
const sessionConfig = context?.perLayerInputsSession ?? context?.config?.perLayerInputsSession ?? null;
|
|
876
|
+
if (getPleSplitTablePolicy(sessionConfig)) {
|
|
877
|
+
return true;
|
|
878
|
+
}
|
|
879
|
+
|
|
875
880
|
const perLayerInputWeights = context?.weights?.get?.('per_layer_inputs');
|
|
876
881
|
if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
|
|
877
882
|
return false;
|
|
@@ -194,11 +194,10 @@ export async function runProbes(stage, buffer, options) {
|
|
|
194
194
|
|
|
195
195
|
const diagnostics = options.operatorDiagnostics ?? null;
|
|
196
196
|
// TSIR fixture writer: when operatorDiagnostics.tsirFixture.dir is
|
|
197
|
-
// set and the stage maps to
|
|
198
|
-
//
|
|
199
|
-
//
|
|
200
|
-
//
|
|
201
|
-
// Doppler reference fixture that the Doe rung ladder consumes.
|
|
197
|
+
// set and the stage maps to a Doppler-to-CSL handoff or TSIR
|
|
198
|
+
// boundary point, capture the activation tensor as a .npy file under
|
|
199
|
+
// <dir>/layer_<layerIdx>/<tsirStage>.npy for Doe validators and
|
|
200
|
+
// splice receipts.
|
|
202
201
|
if (diagnostics?.tsirFixture?.dir) {
|
|
203
202
|
const tsirRecord = await maybeWriteFixtureSnapshot(stage, buffer, {
|
|
204
203
|
tsirFixture: diagnostics.tsirFixture,
|
|
@@ -26,7 +26,7 @@ export interface ResolvedSamplingConfig {
|
|
|
26
26
|
|
|
27
27
|
/**
|
|
28
28
|
* Per-call sampling options. All fields are optional; missing fields fall
|
|
29
|
-
* through to runtime config defaults
|
|
29
|
+
* through to resolved runtime config defaults.
|
|
30
30
|
*/
|
|
31
31
|
export interface SamplingCallOptions {
|
|
32
32
|
temperature?: number;
|
|
@@ -39,16 +39,10 @@ export interface SamplingCallOptions {
|
|
|
39
39
|
* Merge per-call sampling options with runtime config defaults.
|
|
40
40
|
* Returns a complete, validated sampling config object.
|
|
41
41
|
*
|
|
42
|
-
* Precedence: per-call opts > runtimeConfig.inference.sampling
|
|
43
|
-
*
|
|
44
|
-
* Invalid values are clamped to defaults with a warning log rather than
|
|
45
|
-
* throwing, so this is a non-breaking addition.
|
|
46
|
-
*
|
|
47
|
-
* @param opts - Per-call options (temperature, topK, topP, etc.)
|
|
48
|
-
* @param runtimeConfig - Runtime config with inference.sampling defaults
|
|
49
|
-
* @returns Complete sampling config with all fields defined
|
|
42
|
+
* Precedence: per-call opts > runtimeConfig.inference.sampling.
|
|
43
|
+
* Missing runtime fields and invalid call/runtime values throw before execution.
|
|
50
44
|
*/
|
|
51
45
|
export function resolveSamplingConfig(
|
|
52
46
|
opts: SamplingCallOptions | null | undefined,
|
|
53
|
-
runtimeConfig: { inference?: { sampling?:
|
|
47
|
+
runtimeConfig: { inference?: { sampling?: ResolvedSamplingConfig } } | null | undefined
|
|
54
48
|
): ResolvedSamplingConfig;
|
|
@@ -1,79 +1,118 @@
|
|
|
1
|
-
|
|
1
|
+
const REQUIRED_RUNTIME_SAMPLING_FIELDS = [
|
|
2
|
+
'temperature',
|
|
3
|
+
'topP',
|
|
4
|
+
'topK',
|
|
5
|
+
'repetitionPenalty',
|
|
6
|
+
'repetitionPenaltyWindow',
|
|
7
|
+
'greedyThreshold',
|
|
8
|
+
'suppressSpecialTokens',
|
|
9
|
+
'suppressSpecialLikeTokens',
|
|
10
|
+
'suppressTokenIds',
|
|
11
|
+
];
|
|
2
12
|
|
|
3
|
-
const DEFAULT_SAMPLING_DEFAULTS = Object.freeze({
|
|
4
|
-
temperature: 1.0,
|
|
5
|
-
topP: 0.95,
|
|
6
|
-
topK: 50,
|
|
7
|
-
repetitionPenalty: 1.1,
|
|
8
|
-
greedyThreshold: 0.01,
|
|
9
|
-
repetitionPenaltyWindow: 100,
|
|
10
|
-
});
|
|
11
|
-
|
|
12
|
-
/**
|
|
13
|
-
* Merge per-call sampling options with runtime config defaults.
|
|
14
|
-
* Returns a complete, validated sampling config object.
|
|
15
|
-
*
|
|
16
|
-
* Precedence: per-call opts > runtimeConfig.inference.sampling > schema defaults.
|
|
17
|
-
*
|
|
18
|
-
* Invalid values are clamped to defaults with a warning log rather than
|
|
19
|
-
* throwing, so this is a non-breaking addition.
|
|
20
|
-
*
|
|
21
|
-
* @param {Object} opts - Per-call options (temperature, topK, topP, etc.)
|
|
22
|
-
* @param {Object} runtimeConfig - Runtime config with inference.sampling defaults
|
|
23
|
-
* @returns {Object} Complete sampling config with all fields defined
|
|
24
|
-
*/
|
|
25
13
|
export function resolveSamplingConfig(opts, runtimeConfig) {
|
|
26
|
-
const samplingDefaults = runtimeConfig?.inference?.sampling
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
};
|
|
14
|
+
const samplingDefaults = runtimeConfig?.inference?.sampling;
|
|
15
|
+
if (!samplingDefaults || typeof samplingDefaults !== 'object' || Array.isArray(samplingDefaults)) {
|
|
16
|
+
throw new Error('[Sampling] runtimeConfig.inference.sampling is required.');
|
|
17
|
+
}
|
|
18
|
+
for (const field of REQUIRED_RUNTIME_SAMPLING_FIELDS) {
|
|
19
|
+
if (samplingDefaults[field] === undefined) {
|
|
20
|
+
throw new Error(`[Sampling] runtimeConfig.inference.sampling.${field} is required.`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
36
23
|
|
|
37
|
-
|
|
38
|
-
temperature:
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
24
|
+
return {
|
|
25
|
+
temperature: resolveSamplingNumber(
|
|
26
|
+
'temperature',
|
|
27
|
+
opts?.temperature,
|
|
28
|
+
samplingDefaults.temperature,
|
|
29
|
+
(value) => value >= 0
|
|
30
|
+
),
|
|
31
|
+
topP: resolveSamplingNumber(
|
|
32
|
+
'topP',
|
|
33
|
+
opts?.topP,
|
|
34
|
+
samplingDefaults.topP,
|
|
35
|
+
(value) => value > 0 && value <= 1
|
|
36
|
+
),
|
|
37
|
+
topK: resolveSamplingInteger(
|
|
38
|
+
'topK',
|
|
39
|
+
opts?.topK,
|
|
40
|
+
samplingDefaults.topK,
|
|
41
|
+
(value) => value >= 1
|
|
42
|
+
),
|
|
43
|
+
repetitionPenalty: resolveSamplingNumber(
|
|
44
|
+
'repetitionPenalty',
|
|
45
|
+
opts?.repetitionPenalty,
|
|
46
|
+
samplingDefaults.repetitionPenalty,
|
|
47
|
+
(value) => value > 0
|
|
48
|
+
),
|
|
49
|
+
repetitionPenaltyWindow: resolveSamplingInteger(
|
|
50
|
+
'repetitionPenaltyWindow',
|
|
51
|
+
undefined,
|
|
52
|
+
samplingDefaults.repetitionPenaltyWindow,
|
|
53
|
+
(value) => value >= 0
|
|
54
|
+
),
|
|
55
|
+
greedyThreshold: resolveSamplingNumber(
|
|
56
|
+
'greedyThreshold',
|
|
57
|
+
undefined,
|
|
58
|
+
samplingDefaults.greedyThreshold,
|
|
59
|
+
(value) => value >= 0
|
|
60
|
+
),
|
|
61
|
+
suppressSpecialTokens: resolveSamplingBoolean(
|
|
62
|
+
'suppressSpecialTokens',
|
|
63
|
+
samplingDefaults.suppressSpecialTokens
|
|
64
|
+
),
|
|
65
|
+
suppressSpecialLikeTokens: resolveSamplingBoolean(
|
|
66
|
+
'suppressSpecialLikeTokens',
|
|
67
|
+
samplingDefaults.suppressSpecialLikeTokens
|
|
68
|
+
),
|
|
69
|
+
suppressTokenIds: resolveSamplingTokenIdList(
|
|
70
|
+
'suppressTokenIds',
|
|
71
|
+
samplingDefaults.suppressTokenIds
|
|
72
|
+
),
|
|
44
73
|
};
|
|
74
|
+
}
|
|
45
75
|
|
|
46
|
-
|
|
47
|
-
if (
|
|
48
|
-
|
|
76
|
+
function resolveSamplingBoolean(name, runtimeValue) {
|
|
77
|
+
if (runtimeValue === null) {
|
|
78
|
+
throw new Error(`[Sampling] ${name} cannot be null.`);
|
|
79
|
+
}
|
|
80
|
+
if (typeof runtimeValue !== 'boolean') {
|
|
81
|
+
throw new Error(`[Sampling] ${name} must be a boolean; got ${typeof runtimeValue}.`);
|
|
49
82
|
}
|
|
83
|
+
return runtimeValue;
|
|
84
|
+
}
|
|
50
85
|
|
|
51
|
-
|
|
86
|
+
function resolveSamplingTokenIdList(name, runtimeValue) {
|
|
87
|
+
if (!Array.isArray(runtimeValue)) {
|
|
88
|
+
throw new Error(`[Sampling] ${name} must be an array of token IDs.`);
|
|
89
|
+
}
|
|
90
|
+
return runtimeValue.map((tokenId, index) => {
|
|
91
|
+
if (!Number.isInteger(tokenId) || tokenId < 0) {
|
|
92
|
+
throw new Error(`[Sampling] ${name}[${index}] must be a non-negative integer token ID; got ${tokenId}.`);
|
|
93
|
+
}
|
|
94
|
+
return tokenId;
|
|
95
|
+
});
|
|
52
96
|
}
|
|
53
97
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
* @param {number} value - Raw value
|
|
60
|
-
* @param {number} min - Minimum allowed value (inclusive)
|
|
61
|
-
* @param {number} max - Maximum allowed value (inclusive)
|
|
62
|
-
* @param {number} fallback - Default value to use if clamping fails
|
|
63
|
-
* @returns {number} Clamped value
|
|
64
|
-
*/
|
|
65
|
-
function clampWithWarning(name, value, min, max, fallback) {
|
|
98
|
+
function resolveSamplingNumber(name, callValue, runtimeValue, validate) {
|
|
99
|
+
const value = callValue === undefined ? runtimeValue : callValue;
|
|
100
|
+
if (value === null) {
|
|
101
|
+
throw new Error(`[Sampling] ${name} cannot be null.`);
|
|
102
|
+
}
|
|
66
103
|
if (!Number.isFinite(value)) {
|
|
67
|
-
|
|
68
|
-
return fallback;
|
|
104
|
+
throw new Error(`[Sampling] ${name} must be a finite number; got ${value}.`);
|
|
69
105
|
}
|
|
70
|
-
if (value
|
|
71
|
-
|
|
72
|
-
return min;
|
|
106
|
+
if (!validate(value)) {
|
|
107
|
+
throw new Error(`[Sampling] ${name} is outside the configured range; got ${value}.`);
|
|
73
108
|
}
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
109
|
+
return value;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function resolveSamplingInteger(name, callValue, runtimeValue, validate) {
|
|
113
|
+
const value = resolveSamplingNumber(name, callValue, runtimeValue, validate);
|
|
114
|
+
if (Math.floor(value) !== value) {
|
|
115
|
+
throw new Error(`[Sampling] ${name} must be an integer; got ${value}.`);
|
|
77
116
|
}
|
|
78
117
|
return value;
|
|
79
118
|
}
|
|
@@ -58,12 +58,9 @@ export function softmax(logits) {
|
|
|
58
58
|
return exps;
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
-
function countFiniteCandidates(logits
|
|
61
|
+
function countFiniteCandidates(logits) {
|
|
62
62
|
let finiteCandidateCount = 0;
|
|
63
63
|
for (let i = 0; i < logits.length; i++) {
|
|
64
|
-
if (padTokenId != null && i === padTokenId) {
|
|
65
|
-
continue;
|
|
66
|
-
}
|
|
67
64
|
if (Number.isFinite(logits[i])) {
|
|
68
65
|
finiteCandidateCount += 1;
|
|
69
66
|
}
|
|
@@ -71,13 +68,13 @@ function countFiniteCandidates(logits, padTokenId) {
|
|
|
71
68
|
return finiteCandidateCount;
|
|
72
69
|
}
|
|
73
70
|
|
|
74
|
-
function assertFiniteSamplingCandidates(logits,
|
|
75
|
-
const finiteCandidateCount = countFiniteCandidates(logits
|
|
71
|
+
function assertFiniteSamplingCandidates(logits, label) {
|
|
72
|
+
const finiteCandidateCount = countFiniteCandidates(logits);
|
|
76
73
|
if (finiteCandidateCount > 0) {
|
|
77
74
|
return;
|
|
78
75
|
}
|
|
79
76
|
throw new Error(
|
|
80
|
-
`[Sampling] ${label} has no finite candidate logits after masking
|
|
77
|
+
`[Sampling] ${label} has no finite candidate logits after masking suppressed tokens. ` +
|
|
81
78
|
'Upstream decode likely produced NaN/Inf or an all-masked distribution.'
|
|
82
79
|
);
|
|
83
80
|
}
|
|
@@ -217,13 +214,20 @@ function sampleFromLogitCandidates(candidates, temperature, seed, decode, debug,
|
|
|
217
214
|
|
|
218
215
|
|
|
219
216
|
export function sample(logits, opts) {
|
|
220
|
-
const { temperature, topP, topK, decode, debug = false, padTokenId, seed } = opts;
|
|
217
|
+
const { temperature, topP, topK, decode, debug = false, padTokenId, seed, suppressTokenIds } = opts;
|
|
221
218
|
|
|
222
219
|
if (padTokenId !== undefined && padTokenId >= 0 && padTokenId < logits.length) {
|
|
223
220
|
logits[padTokenId] = -Infinity;
|
|
224
221
|
}
|
|
222
|
+
if (Array.isArray(suppressTokenIds)) {
|
|
223
|
+
for (const tokenId of suppressTokenIds) {
|
|
224
|
+
if (Number.isInteger(tokenId) && tokenId >= 0 && tokenId < logits.length) {
|
|
225
|
+
logits[tokenId] = -Infinity;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
225
229
|
|
|
226
|
-
assertFiniteSamplingCandidates(logits,
|
|
230
|
+
assertFiniteSamplingCandidates(logits, 'Logits');
|
|
227
231
|
|
|
228
232
|
// Greedy (argmax) when temperature = 0
|
|
229
233
|
if (temperature === 0) {
|
|
@@ -94,14 +94,22 @@ export class PipelineState {
|
|
|
94
94
|
prefillTimeMs: 0,
|
|
95
95
|
decodeTimeMs: 0,
|
|
96
96
|
ttftMs: 0,
|
|
97
|
+
loadTiming: null,
|
|
98
|
+
pipelineLoadTiming: null,
|
|
97
99
|
prefillTokens: 0,
|
|
98
100
|
decodeTokens: 0,
|
|
99
101
|
memoryUsageBytes: 0,
|
|
100
102
|
tokensGenerated: 0,
|
|
101
103
|
totalTimeMs: 0,
|
|
102
104
|
decodeRecordMs: 0,
|
|
105
|
+
decodeRecordOps: 0,
|
|
106
|
+
decodeRecordPasses: 0,
|
|
107
|
+
decodeRecordOpLabels: {},
|
|
103
108
|
decodeSubmitWaitMs: 0,
|
|
104
109
|
decodeReadbackWaitMs: 0,
|
|
110
|
+
decodeReadbackMapWaitMs: 0,
|
|
111
|
+
decodeReadbackCleanupMs: 0,
|
|
112
|
+
decodeReadbackCopyMs: 0,
|
|
105
113
|
prefillRecordMs: 0,
|
|
106
114
|
prefillSubmitWaitMs: 0,
|
|
107
115
|
prefillProfileSteps: [],
|
|
@@ -115,6 +123,9 @@ export class PipelineState {
|
|
|
115
123
|
batchGuardReason: null,
|
|
116
124
|
singleTokenSubmitWaitMs: 0,
|
|
117
125
|
singleTokenReadbackWaitMs: 0,
|
|
126
|
+
singleTokenReadbackMapWaitMs: 0,
|
|
127
|
+
singleTokenReadbackCleanupMs: 0,
|
|
128
|
+
singleTokenReadbackCopyMs: 0,
|
|
118
129
|
singleTokenOrchestrationMs: 0,
|
|
119
130
|
plePreparedTokenCacheHits: 0,
|
|
120
131
|
plePreparedTokenCacheMisses: 0,
|
|
@@ -131,6 +142,12 @@ export class PipelineState {
|
|
|
131
142
|
totalBatchedTimeMs: 0,
|
|
132
143
|
totalUnbatchedTimeMs: 0,
|
|
133
144
|
gpuSubmissions: 0,
|
|
145
|
+
requestedBatchTokens: 0,
|
|
146
|
+
effectiveBatchTokens: 0,
|
|
147
|
+
executedBatchTokens: 0,
|
|
148
|
+
resolvedBatchTokens: 0,
|
|
149
|
+
maxBatchTokenCap: null,
|
|
150
|
+
batchClampCount: 0,
|
|
134
151
|
};
|
|
135
152
|
|
|
136
153
|
// Base URL for loading assets
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
export interface TsirFixture {
|
|
2
|
+
dir: string;
|
|
3
|
+
layerFilter?: number[] | null;
|
|
4
|
+
prefillOnly?: boolean;
|
|
5
|
+
pendingReads?: TsirPendingRead[];
|
|
6
|
+
records?: TsirFixtureRecord[];
|
|
7
|
+
qkvByLayer?: Map<number, unknown>;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface TsirPendingRead {
|
|
11
|
+
stage: string;
|
|
12
|
+
tsirStage: string;
|
|
13
|
+
layerIdx: number;
|
|
14
|
+
numTokens: number;
|
|
15
|
+
hiddenSize: number;
|
|
16
|
+
dtype: 'f16' | 'f32';
|
|
17
|
+
carry: GPUBuffer;
|
|
18
|
+
alignedBytes: number;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface TsirFixtureRecord {
|
|
22
|
+
stage: string;
|
|
23
|
+
tsirStage: string;
|
|
24
|
+
layerIdx: number;
|
|
25
|
+
filePath?: string;
|
|
26
|
+
shape?: number[];
|
|
27
|
+
dtype?: string;
|
|
28
|
+
byteLength?: number;
|
|
29
|
+
payloadByteLength?: number;
|
|
30
|
+
written: boolean;
|
|
31
|
+
note?: string;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export declare const TSIR_BOUNDARY_STAGES: string[];
|
|
35
|
+
|
|
36
|
+
export declare function mapStageToTsirBoundary(stage: string): string | null;
|
|
37
|
+
|
|
38
|
+
export declare function writeNpyF32(
|
|
39
|
+
filePath: string,
|
|
40
|
+
shape: readonly number[],
|
|
41
|
+
data: Float32Array
|
|
42
|
+
): Promise<{ byteLength: number; payloadByteLength: number }>;
|
|
43
|
+
|
|
44
|
+
export declare function maybeWriteFixtureSnapshot(
|
|
45
|
+
stage: string,
|
|
46
|
+
buffer: Float32Array | GPUBuffer,
|
|
47
|
+
options: {
|
|
48
|
+
tsirFixture?: TsirFixture | null;
|
|
49
|
+
layerIdx: number;
|
|
50
|
+
numTokens: number;
|
|
51
|
+
hiddenSize: number;
|
|
52
|
+
dtype?: 'f16' | 'f32';
|
|
53
|
+
recorder?: { getEncoder?(): GPUCommandEncoder } | null;
|
|
54
|
+
}
|
|
55
|
+
): Promise<TsirFixtureRecord | null>;
|
|
56
|
+
|
|
57
|
+
export declare function drainPendingTsirReads(tsirFixture: TsirFixture | null | undefined): Promise<TsirFixtureRecord[]>;
|
|
@@ -1,20 +1,18 @@
|
|
|
1
1
|
// Doppler-side TSIR boundary-point fixture writer.
|
|
2
2
|
//
|
|
3
|
-
// Builds
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// Captures activation snapshots at the four TSIR boundary points
|
|
7
|
-
// (`post_rmsnorm`, `post_qkv`, `post_attn`, `post_ffn`) during a
|
|
3
|
+
// Builds frozen-Doppler-reference fixtures for the Doe Cerebras lane
|
|
4
|
+
// (see /home/x/deco/doe/config/doe-frozen-doppler-reference.schema.json).
|
|
5
|
+
// Captures handoff and TSIR boundary activation snapshots during a
|
|
8
6
|
// reference inference, writes them as `.npy` files keyed by layer
|
|
9
|
-
// index and probe-point name, and emits a manifest the Doe
|
|
10
|
-
//
|
|
11
|
-
// consume.
|
|
7
|
+
// index and probe-point name, and emits a manifest the Doe validator
|
|
8
|
+
// (bench/tools/validate_frozen_doppler_reference.py) can consume.
|
|
12
9
|
//
|
|
13
10
|
// Triggered when `operatorDiagnostics.tsirFixture` is set in the
|
|
14
11
|
// inference context. The CLI flag --tsir-fixture-dir on
|
|
15
12
|
// tools/run-program-bundle-reference.js plumbs it in.
|
|
16
13
|
//
|
|
17
14
|
// Stage name mapping (Doppler stage -> Doe TSIR boundary):
|
|
15
|
+
// layer_in -> pre_layer_input
|
|
18
16
|
// post_input_norm -> post_rmsnorm
|
|
19
17
|
// linear_qkv_proj -> post_qkv (fused QKV path; single matmul)
|
|
20
18
|
// q_proj+k_proj+v_proj -> post_qkv (split QKV path; concatenated along
|
|
@@ -50,6 +48,7 @@ async function getNodeFs() {
|
|
|
50
48
|
}
|
|
51
49
|
|
|
52
50
|
const STAGE_TO_TSIR = {
|
|
51
|
+
layer_in: 'pre_layer_input',
|
|
53
52
|
post_input_norm: 'post_rmsnorm',
|
|
54
53
|
// Two QKV-projection paths exist in Doppler:
|
|
55
54
|
// - Fused: `linear_qkv_proj` (linear-attention.js variants)
|
|
@@ -336,7 +335,7 @@ export async function maybeWriteFixtureSnapshot(stage, buffer, options) {
|
|
|
336
335
|
// name `post_qkv`. For models with split q_proj/k_proj/v_proj (Gemma 4 31B,
|
|
337
336
|
// etc.), we capture each individually for debug introspection AND emit the
|
|
338
337
|
// concatenated Q∥K∥V tensor along the feature axis as `post_qkv.npy` so the
|
|
339
|
-
// Doe-side
|
|
338
|
+
// Doe-side builder, validator, and splice receipts can consume it. The
|
|
340
339
|
// fused-QKV path (`linear_qkv_proj`) writes to `post_qkv.npy` directly and
|
|
341
340
|
// is not re-synthesized.
|
|
342
341
|
export async function drainPendingTsirReads(tsirFixture) {
|