doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
export interface ExecutionPlanSummary {
|
|
2
|
+
id: string;
|
|
3
|
+
kernelPathId: string | null;
|
|
4
|
+
kernelPathSource: string;
|
|
5
|
+
activationDtype: string;
|
|
6
|
+
readbackInterval: number | null;
|
|
7
|
+
readbackMode: string | null;
|
|
8
|
+
batchSize: number | undefined;
|
|
9
|
+
stopCheckMode: string | undefined;
|
|
10
|
+
ringTokens: number | null;
|
|
11
|
+
ringStop: number | null;
|
|
12
|
+
ringStaging: number | null;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export declare function shouldDisableBatchDecodeAfterShortBatch(input: {
|
|
16
|
+
hitStop?: boolean;
|
|
17
|
+
actualCount?: number;
|
|
18
|
+
requestedCount?: number;
|
|
19
|
+
}): boolean;
|
|
20
|
+
|
|
21
|
+
export declare function resolveHotVocabularyBatchDecodeAvailability(input: {
|
|
22
|
+
hasRangeBackedPerLayerInputs?: boolean;
|
|
23
|
+
pleHotVocabularyRuntime?: Record<string, unknown> | null;
|
|
24
|
+
tokenId?: number;
|
|
25
|
+
}): boolean;
|
|
26
|
+
|
|
27
|
+
export declare function primePleDecodeRuntimeCache(
|
|
28
|
+
state: Record<string, unknown>,
|
|
29
|
+
seedTokenIds?: number[] | null
|
|
30
|
+
): Promise<void>;
|
|
31
|
+
|
|
32
|
+
export declare function recordPrefillProfileStep(
|
|
33
|
+
state: Record<string, unknown>,
|
|
34
|
+
entry: Record<string, unknown>
|
|
35
|
+
): void;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Resolve display text for token IDs. Custom primary and fallback renderers
|
|
39
|
+
* are tried around tokenizer.decode, preserving empty-string special-token
|
|
40
|
+
* filtering when fallback text looks like a special token.
|
|
41
|
+
*/
|
|
42
|
+
export declare function resolveTokenText(
|
|
43
|
+
tokenizer: { decode?(ids: readonly number[], skipSpecialTokens?: boolean): string },
|
|
44
|
+
tokenIds: readonly number[],
|
|
45
|
+
fallbackText?: string,
|
|
46
|
+
renderTokenText?: ((ids: readonly number[]) => string) | null,
|
|
47
|
+
renderFallbackTokenText?: ((ids: readonly number[]) => string) | null
|
|
48
|
+
): string;
|
|
49
|
+
|
|
50
|
+
export declare function usesReplayPrefillDecode(state: Record<string, unknown> | null | undefined): boolean;
|
|
51
|
+
|
|
52
|
+
export declare function assertIncrementalDecodeSupport(
|
|
53
|
+
state: Record<string, unknown>,
|
|
54
|
+
operation: string
|
|
55
|
+
): void;
|
|
56
|
+
|
|
57
|
+
export declare function summarizeExecutionPlan(plan: Record<string, unknown> | null | undefined): ExecutionPlanSummary | null;
|
|
58
|
+
|
|
59
|
+
export declare function shouldRetryWithFinitenessFallback(error: unknown): boolean;
|
|
60
|
+
|
|
61
|
+
export declare function createUnhandledFinitenessPolicyError(
|
|
62
|
+
state: Record<string, unknown>,
|
|
63
|
+
contextLabel: string,
|
|
64
|
+
error: unknown
|
|
65
|
+
): Error;
|
|
66
|
+
|
|
67
|
+
export declare function resolveTargetPlanKVDtype(
|
|
68
|
+
plan: Record<string, unknown> | null | undefined,
|
|
69
|
+
contextLabel: string
|
|
70
|
+
): 'f16' | 'f32';
|
|
71
|
+
|
|
72
|
+
export declare function resolveCurrentKVCacheDtype(
|
|
73
|
+
state: Record<string, unknown>,
|
|
74
|
+
plan: Record<string, unknown> | null | undefined,
|
|
75
|
+
contextLabel: string
|
|
76
|
+
): 'f16' | 'f32';
|
|
77
|
+
|
|
78
|
+
export declare function cloneRuntimeInferenceWithKVDtype(
|
|
79
|
+
state: Record<string, unknown>,
|
|
80
|
+
kvDtype: 'f16' | 'f32'
|
|
81
|
+
): Record<string, unknown>;
|
|
@@ -86,18 +86,6 @@ export function recordPrefillProfileStep(state, entry) {
|
|
|
86
86
|
state.stats.prefillProfileSteps.push(entry);
|
|
87
87
|
}
|
|
88
88
|
|
|
89
|
-
/**
|
|
90
|
-
* Resolve display text for a token ID sequence.
|
|
91
|
-
*
|
|
92
|
-
* Preference order:
|
|
93
|
-
* 1. renderTokenText (custom primary renderer) — if provided and returns non-empty string.
|
|
94
|
-
* 2. tokenizer.decode (default primary) — if renderTokenText is not provided.
|
|
95
|
-
* 3. renderFallbackTokenText (custom fallback) — if provided and returns non-empty string,
|
|
96
|
-
* unless primary returned empty string and fallback looks like a special token (<...>),
|
|
97
|
-
* in which case empty string is preserved to keep skip-special behavior deterministic.
|
|
98
|
-
* 4. tokenizer.decode with skipSpecialTokens=false (default fallback) — same special-token guard.
|
|
99
|
-
* 5. fallbackText — static fallback string (default '?').
|
|
100
|
-
*/
|
|
101
89
|
export function resolveTokenText(tokenizer, tokenIds, fallbackText = '?', renderTokenText, renderFallbackTokenText) {
|
|
102
90
|
const renderPrimary = typeof renderTokenText === 'function'
|
|
103
91
|
? renderTokenText
|
|
@@ -162,8 +150,10 @@ export function summarizeExecutionPlan(plan) {
|
|
|
162
150
|
kernelPathSource: plan.kernelPathSource ?? 'none',
|
|
163
151
|
activationDtype: plan.activationDtype,
|
|
164
152
|
readbackInterval: plan.readbackInterval ?? null,
|
|
153
|
+
readbackMode: plan.readbackMode ?? null,
|
|
165
154
|
batchSize: plan.defaultBatchSize,
|
|
166
155
|
stopCheckMode: plan.defaultStopCheckMode,
|
|
156
|
+
disableCommandBatching: plan.defaultDisableCommandBatching === true,
|
|
167
157
|
ringTokens: plan.ringTokens ?? null,
|
|
168
158
|
ringStop: plan.ringStop ?? null,
|
|
169
159
|
ringStaging: plan.ringStaging ?? null,
|
|
@@ -5,7 +5,7 @@ import { log } from '../../../debug/index.js';
|
|
|
5
5
|
import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
|
|
6
6
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
7
7
|
import { decodeReadback } from './debug-utils/index.js';
|
|
8
|
-
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
|
|
8
|
+
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
9
9
|
import { resolveRangeAwareSelectiveWideningConfig } from './finiteness-policy.js';
|
|
10
10
|
import { resolveActiveExecutionPlan } from './execution-plan.js';
|
|
11
11
|
import { isPlainObject } from '../../../utils/plain-object.js';
|
|
@@ -123,6 +123,9 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
|
|
|
123
123
|
kvCache: state.kvCache,
|
|
124
124
|
currentSeqLen: state.currentSeqLen,
|
|
125
125
|
currentTokenIds: state.currentTokenIds ?? null,
|
|
126
|
+
diffusionGemmaDecoder: false,
|
|
127
|
+
__precomputedInputNorm: null,
|
|
128
|
+
__postFfnNextInputNorm: null,
|
|
126
129
|
useGPU: state.useGPU,
|
|
127
130
|
debug: state.debug,
|
|
128
131
|
stats: state.stats,
|
|
@@ -202,7 +205,7 @@ export function getLogitsWeights(state) {
|
|
|
202
205
|
if (!finalNorm || !(isGpuBufferInstance(finalNorm) || finalNorm instanceof Float32Array)) {
|
|
203
206
|
throw new Error('Final norm not found or invalid type');
|
|
204
207
|
}
|
|
205
|
-
if (!lmHead || !(isGpuBufferInstance(lmHead) || lmHead instanceof Float32Array || isWeightBuffer(lmHead) || isCpuWeightBuffer(lmHead))) {
|
|
208
|
+
if (!lmHead || !(isGpuBufferInstance(lmHead) || lmHead instanceof Float32Array || isWeightBuffer(lmHead) || isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead))) {
|
|
206
209
|
throw new Error('LM head not found or invalid type');
|
|
207
210
|
}
|
|
208
211
|
return { finalNorm, lmHead };
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import type { Tensor } from '../../../gpu/tensor.js';
|
|
2
|
+
|
|
3
|
+
export interface StructuredPromptRequest {
|
|
4
|
+
messages: unknown[];
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export interface PrefixEmbeddingOverride {
|
|
8
|
+
prefixLength: number;
|
|
9
|
+
offset: number;
|
|
10
|
+
embeddings: Float32Array | GPUBuffer;
|
|
11
|
+
expectedLength: number;
|
|
12
|
+
byteLength: number;
|
|
13
|
+
byteOffset: number;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface InputSpan {
|
|
17
|
+
offset: number;
|
|
18
|
+
length: number;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export type PrefixEmbeddingOverrideTransitionDeclaration =
|
|
22
|
+
| 'step_precision'
|
|
23
|
+
| 'explicit_cast_step'
|
|
24
|
+
| null;
|
|
25
|
+
|
|
26
|
+
export interface PrefixEmbeddingOverrideExecutionOptions {
|
|
27
|
+
executionPolicies?: Record<string, unknown> | null;
|
|
28
|
+
transitionDeclaredBy?: PrefixEmbeddingOverrideTransitionDeclaration;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export declare function resolvePromptInput(
|
|
32
|
+
state: Record<string, unknown>,
|
|
33
|
+
prompt: string | unknown[] | StructuredPromptRequest,
|
|
34
|
+
useChatTemplate: boolean,
|
|
35
|
+
contextLabel: string
|
|
36
|
+
): string;
|
|
37
|
+
|
|
38
|
+
export declare function resolveEffectivePrefillTokenChunkSize(
|
|
39
|
+
state: Record<string, unknown>
|
|
40
|
+
): number | null | undefined;
|
|
41
|
+
|
|
42
|
+
export declare function releasePerLayerInputBuffer(
|
|
43
|
+
buffer: GPUBuffer | null | undefined,
|
|
44
|
+
recorder: { trackTemporaryBuffer(buffer: GPUBuffer): void } | null | undefined,
|
|
45
|
+
decodeBuffers: { ownsBuffer(buffer: GPUBuffer): boolean } | null | undefined,
|
|
46
|
+
pleCache?: { ownedBuffers?: Set<GPUBuffer> } | null
|
|
47
|
+
): void;
|
|
48
|
+
|
|
49
|
+
export declare function normalizePrefixEmbeddingOverride(
|
|
50
|
+
override: Record<string, unknown> | null | undefined,
|
|
51
|
+
hiddenSize: number,
|
|
52
|
+
numTokens: number,
|
|
53
|
+
contextLabel: string
|
|
54
|
+
): PrefixEmbeddingOverride | null;
|
|
55
|
+
|
|
56
|
+
export declare function resolvePrefillEmbeddingInputIds(
|
|
57
|
+
inputIds: readonly number[],
|
|
58
|
+
embeddingInputSpan: InputSpan & { tokenId: number } | null | undefined,
|
|
59
|
+
contextLabel: string
|
|
60
|
+
): number[] | readonly number[];
|
|
61
|
+
|
|
62
|
+
export declare function resolvePrefillMultimodalBidirectionalSpan(
|
|
63
|
+
inputIds: readonly number[],
|
|
64
|
+
bidirectionalSpan: InputSpan | null | undefined,
|
|
65
|
+
contextLabel: string
|
|
66
|
+
): InputSpan | null;
|
|
67
|
+
|
|
68
|
+
export declare function resolvePrefixEmbeddingOverrideTransitionDeclaredBy(
|
|
69
|
+
executionV1State: Record<string, unknown> | null | undefined
|
|
70
|
+
): PrefixEmbeddingOverrideTransitionDeclaration;
|
|
71
|
+
|
|
72
|
+
export declare function applyPrefixEmbeddingOverride(
|
|
73
|
+
baseTensor: Tensor,
|
|
74
|
+
override: PrefixEmbeddingOverride | null,
|
|
75
|
+
hiddenSize: number,
|
|
76
|
+
contextLabel: string,
|
|
77
|
+
executionOptions?: Record<string, unknown> | PrefixEmbeddingOverrideExecutionOptions | null
|
|
78
|
+
): Promise<Tensor>;
|
|
79
|
+
|
|
80
|
+
export declare function shouldDisablePrefillCommandBatching(
|
|
81
|
+
state: Record<string, unknown>,
|
|
82
|
+
opts: Record<string, unknown>,
|
|
83
|
+
multimodalBidirectionalSpan: InputSpan | null
|
|
84
|
+
): boolean;
|
|
@@ -16,6 +16,50 @@ function isStructuredChatRequest(prompt) {
|
|
|
16
16
|
&& Array.isArray(prompt.messages);
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
+
function normalizePrefixEmbeddingOverrideExecutionOptions(options) {
|
|
20
|
+
if (
|
|
21
|
+
options != null
|
|
22
|
+
&& typeof options === 'object'
|
|
23
|
+
&& !Array.isArray(options)
|
|
24
|
+
&& (
|
|
25
|
+
Object.prototype.hasOwnProperty.call(options, 'executionPolicies')
|
|
26
|
+
|| Object.prototype.hasOwnProperty.call(options, 'transitionDeclaredBy')
|
|
27
|
+
)
|
|
28
|
+
) {
|
|
29
|
+
return {
|
|
30
|
+
executionPolicies: options.executionPolicies ?? null,
|
|
31
|
+
transitionDeclaredBy: options.transitionDeclaredBy ?? null,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
return {
|
|
35
|
+
executionPolicies: options ?? null,
|
|
36
|
+
transitionDeclaredBy: null,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function normalizeExecutionDtype(value) {
|
|
41
|
+
const normalized = typeof value === 'string' ? value.trim().toLowerCase() : '';
|
|
42
|
+
return normalized === 'f16' || normalized === 'f32' ? normalized : null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function resolvePrefixEmbeddingOverrideTransitionDeclaredBy(executionV1State) {
|
|
46
|
+
const steps = executionV1State?.resolvedSteps?.all;
|
|
47
|
+
if (!Array.isArray(steps)) {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
for (const step of steps) {
|
|
51
|
+
if (step?.op !== 'cast' || step?.section !== 'preLayer') {
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
const fromDtype = normalizeExecutionDtype(step.fromDtype ?? step.precision?.inputDtype);
|
|
55
|
+
const toDtype = normalizeExecutionDtype(step.toDtype ?? step.precision?.outputDtype);
|
|
56
|
+
if (fromDtype === 'f32' && toDtype === 'f16') {
|
|
57
|
+
return 'explicit_cast_step';
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
|
|
19
63
|
export function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
|
|
20
64
|
const chatOptions = state.modelConfig.chatTemplateThinking === true ? { thinking: true } : undefined;
|
|
21
65
|
if (typeof prompt === 'string') {
|
|
@@ -49,6 +93,19 @@ export function resolvePromptInput(state, prompt, useChatTemplate, contextLabel)
|
|
|
49
93
|
return formatChatMessages(messages, templateType, chatOptions);
|
|
50
94
|
}
|
|
51
95
|
|
|
96
|
+
export function resolveEffectivePrefillTokenChunkSize(state) {
|
|
97
|
+
const runtimeSession = state?.runtimeConfig?.inference?.session;
|
|
98
|
+
const runtimeChunkSize = runtimeSession?.prefillTokenChunkSize;
|
|
99
|
+
if (runtimeChunkSize !== undefined && runtimeChunkSize !== null) {
|
|
100
|
+
return runtimeChunkSize;
|
|
101
|
+
}
|
|
102
|
+
const modelSession = state?.modelConfig?.sessionSettings;
|
|
103
|
+
if (modelSession?.prefillTokenChunkSize !== undefined) {
|
|
104
|
+
return modelSession.prefillTokenChunkSize;
|
|
105
|
+
}
|
|
106
|
+
return runtimeChunkSize;
|
|
107
|
+
}
|
|
108
|
+
|
|
52
109
|
export function releasePerLayerInputBuffer(buffer, recorder, decodeBuffers, pleCache = null) {
|
|
53
110
|
if (!buffer) {
|
|
54
111
|
return;
|
|
@@ -191,10 +248,14 @@ export function resolvePrefillMultimodalBidirectionalSpan(inputIds, bidirectiona
|
|
|
191
248
|
return { offset, length };
|
|
192
249
|
}
|
|
193
250
|
|
|
194
|
-
export async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenSize, contextLabel,
|
|
251
|
+
export async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenSize, contextLabel, executionOptions = null) {
|
|
195
252
|
if (!override) {
|
|
196
253
|
return baseTensor;
|
|
197
254
|
}
|
|
255
|
+
const {
|
|
256
|
+
executionPolicies,
|
|
257
|
+
transitionDeclaredBy,
|
|
258
|
+
} = normalizePrefixEmbeddingOverrideExecutionOptions(executionOptions);
|
|
198
259
|
if (baseTensor.dtype !== 'f32' && baseTensor.dtype !== 'f16') {
|
|
199
260
|
throw new Error(
|
|
200
261
|
`[Pipeline] ${contextLabel}: embedding overrides require f32 or f16 activations, got ${baseTensor.dtype}.`
|
|
@@ -236,6 +297,7 @@ export async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenS
|
|
|
236
297
|
toDtype: 'f16',
|
|
237
298
|
op: 'prefill_embedding_override',
|
|
238
299
|
detail: 'Prefix embedding override would pack GPU-provided f32 features into an f16 activation buffer.',
|
|
300
|
+
transitionDeclaredBy,
|
|
239
301
|
});
|
|
240
302
|
const castedOverride = await castF32ToF16(overrideTensor);
|
|
241
303
|
try {
|
|
@@ -257,6 +319,7 @@ export async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenS
|
|
|
257
319
|
toDtype: 'f16',
|
|
258
320
|
op: 'prefill_embedding_override',
|
|
259
321
|
detail: 'Prefix embedding override would pack CPU-provided f32 features into an f16 activation buffer.',
|
|
322
|
+
transitionDeclaredBy,
|
|
260
323
|
});
|
|
261
324
|
const packedOverride = f32ToF16Array(override.embeddings);
|
|
262
325
|
device.queue.submit([encoder.finish()]);
|
|
@@ -280,6 +343,9 @@ export function shouldDisablePrefillCommandBatching(state, opts, multimodalBidir
|
|
|
280
343
|
if (state?.kvCache?.layout === 'bdpa_paged') {
|
|
281
344
|
return true;
|
|
282
345
|
}
|
|
346
|
+
if (resolveEffectivePrefillTokenChunkSize(state) != null) {
|
|
347
|
+
return true;
|
|
348
|
+
}
|
|
283
349
|
if (multimodalBidirectionalSpan == null) {
|
|
284
350
|
return false;
|
|
285
351
|
}
|
|
@@ -43,8 +43,25 @@ export interface FusedDecodeSamplingConfig {
|
|
|
43
43
|
|
|
44
44
|
export declare function shouldUseFusedDecodeSampling(config: FusedDecodeSamplingConfig): boolean;
|
|
45
45
|
|
|
46
|
+
export interface StopTokenLookup {
|
|
47
|
+
firstTokenId: number | null;
|
|
48
|
+
secondTokenId: number | null;
|
|
49
|
+
tokenSet: Set<number> | null;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export declare function createStopTokenLookup(
|
|
53
|
+
stopTokenIds: number[],
|
|
54
|
+
eosTokenId?: number | null
|
|
55
|
+
): StopTokenLookup;
|
|
56
|
+
|
|
57
|
+
export declare function resolveBatchStop(
|
|
58
|
+
tokens: ArrayLike<number>,
|
|
59
|
+
stopFlags: ArrayLike<number> | null,
|
|
60
|
+
stopTokenLookup: StopTokenLookup
|
|
61
|
+
): number;
|
|
62
|
+
|
|
46
63
|
export declare function findInvalidGeneratedToken(
|
|
47
|
-
tokens: number
|
|
64
|
+
tokens: ArrayLike<number>,
|
|
48
65
|
vocabSize: number,
|
|
49
66
|
padTokenId?: number | null
|
|
50
67
|
): { index: number; tokenId: number } | null;
|
|
@@ -56,12 +73,17 @@ export interface SampledTokenStagingBuffer {
|
|
|
56
73
|
destroy(): void;
|
|
57
74
|
}
|
|
58
75
|
|
|
76
|
+
export interface DeferredCleanupRecorder {
|
|
77
|
+
completeDeferredCleanup(options?: { discardPooled?: boolean }): Promise<void> | void;
|
|
78
|
+
}
|
|
79
|
+
|
|
59
80
|
export declare function readSampledTokenFromStagingBuffer(
|
|
60
81
|
stagingBuffer: SampledTokenStagingBuffer,
|
|
61
82
|
options?: {
|
|
62
83
|
ownsStagingBuffer?: boolean;
|
|
63
84
|
hasFinitenessBuffer?: boolean;
|
|
64
85
|
ring?: { advance(): void } | null;
|
|
86
|
+
cleanupRecorder?: DeferredCleanupRecorder | null;
|
|
65
87
|
}
|
|
66
88
|
): Promise<{
|
|
67
89
|
nextToken: number;
|
|
@@ -69,6 +91,11 @@ export declare function readSampledTokenFromStagingBuffer(
|
|
|
69
91
|
triggered: boolean;
|
|
70
92
|
metadata: string;
|
|
71
93
|
};
|
|
94
|
+
timing: {
|
|
95
|
+
mapWaitMs: number;
|
|
96
|
+
cleanupMs: number;
|
|
97
|
+
copyMs: number;
|
|
98
|
+
};
|
|
72
99
|
}>;
|
|
73
100
|
|
|
74
101
|
export declare function readMappedBufferCopy(
|
|
@@ -82,17 +109,25 @@ export declare function readBatchTokensFromStagingBuffers(options: {
|
|
|
82
109
|
tokensStagingBuffer: SampledTokenStagingBuffer;
|
|
83
110
|
stopStagingBuffer?: SampledTokenStagingBuffer | null;
|
|
84
111
|
finitenessStagingBuffer?: SampledTokenStagingBuffer | null;
|
|
112
|
+
finitenessOffsetBytes?: number | null;
|
|
85
113
|
tokenCount: number;
|
|
86
114
|
ownsTokensStaging?: boolean;
|
|
87
115
|
ownsStopStaging?: boolean;
|
|
116
|
+
ownsFinitenessStaging?: boolean;
|
|
88
117
|
ring?: { advance(): void } | null;
|
|
118
|
+
cleanupRecorder?: DeferredCleanupRecorder | null;
|
|
89
119
|
}): Promise<{
|
|
90
|
-
tokens:
|
|
120
|
+
tokens: Uint32Array;
|
|
91
121
|
stopFlags: Uint32Array | null;
|
|
92
122
|
finitenessStatus: {
|
|
93
123
|
triggered: boolean;
|
|
94
124
|
metadata: string;
|
|
95
125
|
};
|
|
126
|
+
timing: {
|
|
127
|
+
mapWaitMs: number;
|
|
128
|
+
cleanupMs: number;
|
|
129
|
+
copyMs: number;
|
|
130
|
+
};
|
|
96
131
|
}>;
|
|
97
132
|
|
|
98
133
|
export declare function decodeStep(
|
|
@@ -148,4 +183,4 @@ export declare function generateNTokensGPU(
|
|
|
148
183
|
getLogitsWeights: () => unknown;
|
|
149
184
|
getLogitsConfig: () => unknown;
|
|
150
185
|
}
|
|
151
|
-
): Promise<{ tokens: number[]; actualCount: number }>;
|
|
186
|
+
): Promise<{ tokens: number[] | Uint32Array; actualCount: number }>;
|