doppler-gpu 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -577
- package/LICENSE +21 -201
- package/README.md +165 -222
- package/models/catalog.json +241 -33
- package/package.json +99 -86
- package/src/capsule-runtime.d.ts +23 -0
- package/src/capsule-runtime.js +23 -0
- package/src/capsule.d.ts +8 -0
- package/src/capsule.js +4 -0
- package/src/cli/argument-contract.js +200 -0
- package/src/cli/browser-report-output.js +24 -0
- package/src/cli/cli-model-resolution.js +2 -2
- package/src/cli/cli-output.js +10 -0
- package/src/cli/commands/bundle.js +828 -0
- package/src/cli/doppler-cli.js +50 -1457
- package/src/cli/doppler-serve.js +18 -244
- package/src/cli/output/manifest.js +368 -0
- package/src/cli/runtime-profiles.js +1 -1
- package/src/cli/serve/capsule-handler.d.ts +33 -0
- package/src/cli/serve/capsule-handler.js +105 -0
- package/src/cli/serve/capsule-http.js +102 -0
- package/src/cli/serve/receipt.js +269 -0
- package/src/client/capsule-host.browser.js +1 -0
- package/src/client/capsule-host.d.ts +3 -0
- package/src/client/capsule-host.js +1 -0
- package/src/client/doppler-api.browser.js +42 -0
- package/src/client/doppler-api.d.ts +36 -1
- package/src/client/doppler-api.js +40 -1
- package/src/client/doppler-registry.js +15 -7
- package/src/client/electron/index.d.ts +4 -0
- package/src/client/electron/index.js +13 -0
- package/src/client/electron/ipc-contract.d.ts +9 -0
- package/src/client/electron/ipc-contract.js +56 -0
- package/src/client/electron/release-state.d.ts +53 -0
- package/src/client/electron/release-state.js +425 -0
- package/src/client/electron/renderer-runtime.d.ts +19 -0
- package/src/client/electron/renderer-runtime.js +104 -0
- package/src/client/failure-taxonomy.js +1 -28
- package/src/client/fault-injection.js +0 -8
- package/src/client/inspection.js +1 -1
- package/src/client/model-host/index.d.ts +140 -0
- package/src/client/model-host/index.js +548 -0
- package/src/client/model-host/model-evidence.js +255 -0
- package/src/client/model-host/model-session.d.ts +322 -0
- package/src/client/{runtime → model-host}/model-session.js +190 -148
- package/src/client/provider.d.ts +18 -2
- package/src/client/provider.js +26 -3
- package/src/client/receipt.js +30 -72
- package/src/client/runtime/capsule-acquisition.d.ts +18 -0
- package/src/client/runtime/capsule-acquisition.js +103 -0
- package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
- package/src/client/runtime/capsule-adapter-execution.js +63 -0
- package/src/client/runtime/capsule-artifact-source.js +62 -0
- package/src/client/runtime/capsule-embedding.d.ts +29 -0
- package/src/client/runtime/capsule-embedding.js +76 -0
- package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
- package/src/client/runtime/capsule-forecast-program.js +85 -0
- package/src/client/runtime/capsule-forecast.d.ts +17 -0
- package/src/client/runtime/capsule-forecast.js +31 -0
- package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
- package/src/client/runtime/capsule-operation-adapters.js +68 -0
- package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
- package/src/client/runtime/capsule-operation-executor.js +100 -0
- package/src/client/runtime/capsule-program-adapter.js +151 -0
- package/src/client/runtime/capsule-release-authorization.js +25 -0
- package/src/client/runtime/capsule-rerank.d.ts +50 -0
- package/src/client/runtime/capsule-rerank.js +105 -0
- package/src/client/runtime/capsule-session-execution.js +85 -0
- package/src/client/runtime/command-executor.d.ts +15 -0
- package/src/client/runtime/command-executor.js +128 -0
- package/src/client/runtime/composition-root.d.ts +74 -0
- package/src/client/runtime/composition-root.js +344 -0
- package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
- package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
- package/src/client/runtime/index.d.ts +1 -117
- package/src/client/runtime/index.js +1 -433
- package/src/client/runtime/lora.d.ts +14 -0
- package/src/client/runtime/lora.js +19 -1
- package/src/client/runtime/model-manager.js +12 -0
- package/src/client/runtime/model-rerank.js +30 -0
- package/src/client/runtime/model-session.d.ts +1 -221
- package/src/client/runtime/model-source.d.ts +9 -0
- package/src/client/runtime/model-source.js +46 -17
- package/src/client/runtime/node-quickstart-cache.js +1 -1
- package/src/client/runtime/resolution-policy.d.ts +20 -0
- package/src/client/runtime/resolution-policy.js +84 -0
- package/src/client/runtime/resource-binder.d.ts +24 -0
- package/src/client/runtime/resource-binder.js +135 -0
- package/src/client/runtime/scoped-session.d.ts +10 -1
- package/src/client/runtime/scoped-session.js +20 -1
- package/src/client/runtime/session-controller.d.ts +30 -0
- package/src/client/runtime/session-controller.js +170 -0
- package/src/client/runtime/source-runtime.js +1 -1
- package/src/client/runtime/target-selector.d.ts +9 -0
- package/src/client/runtime/target-selector.js +2 -0
- package/src/client/runtime/types.d.ts +1 -0
- package/src/client/runtime/verified-capsule-artifact-store.js +72 -0
- package/src/client/wrap-pipeline-handle.js +3 -9
- package/src/config/backward-registry-loader.js +1 -1
- package/src/config/capsule-adapter-policy.js +30 -0
- package/src/config/capsule-adapters.d.ts +21 -0
- package/src/config/capsule-adapters.js +40 -0
- package/src/config/capsule-adapters.json +9 -0
- package/src/config/capsule-loading.d.ts +2 -0
- package/src/config/capsule-loading.js +14 -0
- package/src/config/capsule-loading.schema.json +11 -0
- package/src/config/capsule-operation.d.ts +19 -0
- package/src/config/capsule-operation.js +58 -0
- package/src/config/capsule-operations.json +18 -0
- package/src/config/capsule-release-contract.d.ts +66 -0
- package/src/config/capsule-release-contract.js +197 -0
- package/src/config/capsule-release-events.d.ts +53 -0
- package/src/config/capsule-release-events.js +172 -0
- package/src/config/capsule-sequence-receipt.js +19 -0
- package/src/config/capsule-serve.d.ts +9 -0
- package/src/config/capsule-serve.js +28 -0
- package/src/config/capsule-serve.schema.json +14 -0
- package/src/config/capsule-signature.d.ts +6 -0
- package/src/config/capsule-signature.js +50 -0
- package/src/config/capsule-v2.d.ts +70 -0
- package/src/config/capsule-v2.js +431 -0
- package/src/config/capsule-v3.d.ts +19 -0
- package/src/config/capsule-v3.js +72 -0
- package/src/config/capsule.d.ts +22 -0
- package/src/config/capsule.js +52 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +41 -1
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +1 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +1 -1
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +59 -7
- package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
- package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
- package/src/config/dtype-transition-contract.js +40 -0
- package/src/config/embedding-contract.js +26 -0
- package/src/config/embedding-reference.js +85 -0
- package/src/config/evidence/source-intake-policy.json +1 -1
- package/src/config/execution-contract-check.d.ts +16 -73
- package/src/config/execution-contract-check.js +7 -429
- package/src/config/forecast-manifest.js +56 -0
- package/src/config/heterogeneous-attention-contract.js +46 -0
- package/src/config/initial-execution-identity.d.ts +58 -0
- package/src/config/initial-execution-identity.js +331 -0
- package/src/config/kernel-path-loader.d.ts +10 -0
- package/src/config/kernel-path-loader.js +23 -8
- package/src/config/kernel-registry-contract.d.ts +25 -0
- package/src/config/kernel-registry-contract.js +46 -0
- package/src/config/kernels/kernel-ref-digests.js +50 -2
- package/src/config/kernels/registry.js +1 -1
- package/src/config/kernels/registry.json +13592 -11377
- package/src/config/lora-layouts.d.ts +10 -0
- package/src/config/lora-layouts.js +17 -0
- package/src/config/lora-layouts.json +9 -0
- package/src/config/manifest-inference-validation.js +575 -0
- package/src/config/merge/execution-patches.js +12 -0
- package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
- package/src/config/merge/value-selection.js +38 -0
- package/src/config/merge.js +43 -11
- package/src/config/model-ir-v2.d.ts +57 -0
- package/src/config/model-ir-v2.js +258 -0
- package/src/config/model-ir.d.ts +49 -0
- package/src/config/model-ir.js +180 -0
- package/src/config/param-validator.js +1 -1
- package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
- package/src/config/platforms/loader.js +2 -1
- package/src/config/production-release-evidence.d.ts +23 -0
- package/src/config/production-release-evidence.js +423 -0
- package/src/config/production-release.js +321 -0
- package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
- package/src/config/required-inference-fields-contract-check.js +2 -2
- package/src/config/rerank-reference.d.ts +28 -0
- package/src/config/rerank-reference.js +89 -0
- package/src/config/revocation-policy.d.ts +65 -0
- package/src/config/revocation-policy.js +290 -0
- package/src/config/revocation-registry.json +11 -0
- package/src/config/revocation-updates.d.ts +68 -0
- package/src/config/revocation-updates.js +440 -0
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
- package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
- package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
- package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
- package/src/config/runtime-merge.js +1 -1
- package/src/config/runtime.js +1 -1
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/conversion-report.schema.d.ts +3 -0
- package/src/config/schema/conversion-report.schema.js +29 -0
- package/src/config/schema/converter.schema.d.ts +1 -0
- package/src/config/schema/debug.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +5 -5
- package/src/config/schema/emulation.schema.js +1 -1
- package/src/config/schema/execution-v1.schema.d.ts +3 -0
- package/src/config/schema/execution-v1.schema.js +19 -0
- package/src/config/schema/harness.schema.d.ts +13 -0
- package/src/config/schema/harness.schema.js +6 -0
- package/src/config/schema/inference.schema.d.ts +21 -2
- package/src/config/schema/kernel-thresholds.schema.js +1 -1
- package/src/config/schema/manifest.schema.d.ts +26 -7
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/program-bundle.schema.d.ts +52 -6
- package/src/config/schema/program-bundle.schema.js +145 -10
- package/src/config/sequence-reference.d.ts +26 -0
- package/src/config/sequence-reference.js +100 -0
- package/src/config/target-plan.d.ts +108 -0
- package/src/config/target-plan.js +411 -0
- package/src/config/training-defaults.js +1 -1
- package/src/config/transforms/capability-transform-resolver.js +4 -105
- package/src/config/transforms/execution-graph/dtype.js +591 -0
- package/src/config/transforms/execution-graph/fusion.js +436 -0
- package/src/config/transforms/execution-graph/validation.js +414 -0
- package/src/config/transforms/execution-graph-transforms.js +13 -1419
- package/src/converter/artifact-identity.js +475 -0
- package/src/converter/conversion-plan.js +1 -1
- package/src/converter/core.js +94 -2096
- package/src/converter/forge-candidate-evaluation.js +271 -0
- package/src/converter/forge-qualification.js +162 -0
- package/src/converter/forge-stages.js +798 -0
- package/src/converter/manifest-builder.js +683 -0
- package/src/converter/quantization-info.js +4 -1
- package/src/converter/quantizer.js +58 -9
- package/src/converter/shard-packer.d.ts +1 -1
- package/src/converter/shard-packer.js +2 -2
- package/src/converter/tensor-transform.js +640 -0
- package/src/converter/tokenizer-contract.js +243 -0
- package/src/converter/tokenizer-manifest.js +243 -0
- package/src/debug/history.js +0 -8
- package/src/debug/index.js +0 -10
- package/src/debug/tensor.js +1 -1
- package/src/debug/trace.js +0 -7
- package/src/experimental/adapters/adapter-manager.js +2 -0
- package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
- package/src/experimental/adapters/adapter-manifest.js +11 -0
- package/src/experimental/adapters/artifact-contract.d.ts +2 -2
- package/src/experimental/adapters/artifact-contract.js +5 -2
- package/src/experimental/adapters/lora-loader.d.ts +10 -1
- package/src/experimental/adapters/lora-loader.js +110 -48
- package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
- package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
- package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
- package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
- package/src/experimental/browser/browser-converter.js +11 -11
- package/src/experimental/diffusion/index.d.ts +1 -1
- package/src/experimental/diffusion/index.js +1 -1
- package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
- package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
- package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
- package/src/experimental/diffusion/vae.js +596 -0
- package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
- package/src/experimental/distribution/shard-delivery/retry.js +392 -0
- package/src/experimental/distribution/shard-delivery/transport.js +831 -0
- package/src/experimental/distribution/shard-delivery.js +4 -1709
- package/src/experimental/energy/index.d.ts +2 -2
- package/src/experimental/energy/index.js +2 -2
- package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
- package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
- package/src/experimental/logits/cpu-output.d.ts +27 -0
- package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
- package/src/experimental/logits/cpu.d.ts +87 -0
- package/src/experimental/logits/cpu.js +95 -0
- package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
- package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
- package/src/experimental/orchestration/index.d.ts +1 -1
- package/src/experimental/orchestration/index.js +1 -1
- package/src/experimental/training/artifacts.js +2 -2
- package/src/experimental/training/checkpoint.js +2 -2
- package/src/experimental/training/distillation/artifacts.js +1 -1
- package/src/experimental/training/distillation/dataset.js +2 -2
- package/src/experimental/training/distillation/eval.js +1 -1
- package/src/experimental/training/distillation/runtime.js +2 -2
- package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
- package/src/experimental/training/distillation/student-fixture/model.js +353 -0
- package/src/experimental/training/distillation/student-fixture.js +2 -469
- package/src/experimental/training/lora/checkpoint.js +746 -0
- package/src/experimental/training/lora/execution.js +550 -0
- package/src/experimental/training/lora/plan.js +387 -0
- package/src/experimental/training/lora/recovery.js +418 -0
- package/src/experimental/training/lora-pipeline.js +7 -1949
- package/src/experimental/training/operator-artifacts.js +2 -2
- package/src/experimental/training/operator-command.js +4 -4
- package/src/experimental/training/runner/evaluation.js +62 -0
- package/src/experimental/training/runner/lifecycle.js +257 -0
- package/src/experimental/training/runner.js +5 -268
- package/src/experimental/training/suite/execution.js +879 -0
- package/src/experimental/training/suite/plan.js +855 -0
- package/src/experimental/training/suite.js +5 -1659
- package/src/experimental/training/workloads.d.ts +5 -5
- package/src/experimental/training/workloads.js +6 -6
- package/src/experimental/turboquant/codebook.js +470 -0
- package/src/formats/quantization-constants.d.ts +11 -0
- package/src/formats/quantization-constants.js +11 -0
- package/src/formats/rdrr/integrity.js +1 -1
- package/src/formats/rdrr/lowerings.js +0 -29
- package/src/formats/rdrr/merkle.js +1 -1
- package/src/formats/rdrr/tensor-config-validator.js +25 -5
- package/src/formats/rdrr/validation.js +3 -3
- package/src/gpu/command-recorder.d.ts +10 -1
- package/src/gpu/command-recorder.js +58 -14
- package/src/gpu/constant-buffer.js +29 -0
- package/src/gpu/device-state.js +36 -0
- package/src/gpu/device.js +6 -41
- package/src/gpu/kernel-runtime.js +2 -1
- package/src/gpu/kernels/activation-static-qdq.js +1 -1
- package/src/gpu/kernels/append_embedding.wgsl +14 -0
- package/src/gpu/kernels/attention/executors.js +960 -0
- package/src/gpu/kernels/attention/plan.js +472 -0
- package/src/gpu/kernels/attention.d.ts +4 -0
- package/src/gpu/kernels/attention.js +145 -1545
- package/src/gpu/kernels/attention_relative.wgsl +59 -0
- package/src/gpu/kernels/backward/adam.js +2 -1
- package/src/gpu/kernels/backward/attention_backward.js +1 -1
- package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
- package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
- package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
- package/src/gpu/kernels/backward/embed_backward.js +2 -1
- package/src/gpu/kernels/backward/executor.d.ts +34 -0
- package/src/gpu/kernels/backward/executor.js +158 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
- package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/gelu_backward.js +1 -1
- package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
- package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
- package/src/gpu/kernels/backward/matmul.js +133 -0
- package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
- package/src/gpu/kernels/backward/matmul_backward.js +1 -1
- package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
- package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
- package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/rope_backward.js +2 -1
- package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/scale_backward.js +2 -1
- package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/silu_backward.js +1 -1
- package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/softmax_backward.js +1 -1
- package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
- package/src/gpu/kernels/backward/utils.js +2 -392
- package/src/gpu/kernels/bind-group-validation.js +16 -0
- package/src/gpu/kernels/cast.js +2 -1
- package/src/gpu/kernels/check-finiteness.js +13 -49
- package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
- package/src/gpu/kernels/check-stop.js +14 -33
- package/src/gpu/kernels/check_finiteness.wgsl +33 -0
- package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
- package/src/gpu/kernels/check_stop.wgsl +20 -0
- package/src/gpu/kernels/clamp.js +1 -1
- package/src/gpu/kernels/conv2d.js +1 -1
- package/src/gpu/kernels/cross_entropy_loss.js +1 -1
- package/src/gpu/kernels/depthwise_conv1d.js +1 -1
- package/src/gpu/kernels/depthwise_conv2d.js +1 -1
- package/src/gpu/kernels/dequant.js +7 -2
- package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
- package/src/gpu/kernels/dispatch.js +12 -1
- package/src/gpu/kernels/embedding-pool.js +72 -0
- package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
- package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
- package/src/gpu/kernels/energy.js +2 -1
- package/src/gpu/kernels/forecast_output.wgsl +17 -0
- package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
- package/src/gpu/kernels/fused_ffn.js +10 -220
- package/src/gpu/kernels/fused_matmul_residual.js +2 -1
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
- package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
- package/src/gpu/kernels/gated-short-conv.js +11 -69
- package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
- package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
- package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
- package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
- package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
- package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
- package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
- package/src/gpu/kernels/gather.d.ts +1 -1
- package/src/gpu/kernels/gather.js +30 -6
- package/src/gpu/kernels/gather_bf16.wgsl +69 -0
- package/src/gpu/kernels/gelu.js +1 -1
- package/src/gpu/kernels/gemma4-route-expert.js +1 -1
- package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
- package/src/gpu/kernels/groupnorm.js +1 -1
- package/src/gpu/kernels/index.d.ts +38 -5
- package/src/gpu/kernels/index.js +18 -4
- package/src/gpu/kernels/instance_stats.wgsl +26 -0
- package/src/gpu/kernels/kernel-base.js +1 -1
- package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
- package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
- package/src/gpu/kernels/kernel-configs.d.ts +10 -61
- package/src/gpu/kernels/kernel-configs.js +26 -54
- package/src/gpu/kernels/kernel-execution.js +141 -0
- package/src/gpu/kernels/kv-cache-write.js +4 -4
- package/src/gpu/kernels/kv-quantize.js +4 -1
- package/src/gpu/kernels/l2-normalize.js +36 -0
- package/src/gpu/kernels/l2_normalize.wgsl +53 -0
- package/src/gpu/kernels/layernorm.d.ts +4 -4
- package/src/gpu/kernels/layernorm.js +44 -7
- package/src/gpu/kernels/linear-attention-core.js +29 -449
- package/src/gpu/kernels/linear_attention.js +1 -1
- package/src/gpu/kernels/lm-head-argmax.js +2 -2
- package/src/gpu/kernels/lm-head-select-logits.js +2 -2
- package/src/gpu/kernels/logit-finalize.js +103 -0
- package/src/gpu/kernels/logit-merge.js +13 -125
- package/src/gpu/kernels/logit-scatter.js +40 -0
- package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
- package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
- package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
- package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
- package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
- package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
- package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
- package/src/gpu/kernels/matmul-dispatch.js +6 -1
- package/src/gpu/kernels/matmul-selection.js +3 -2
- package/src/gpu/kernels/matmul.js +4 -15
- package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
- package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
- package/src/gpu/kernels/modulate.js +1 -1
- package/src/gpu/kernels/moe.js +3 -1
- package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
- package/src/gpu/kernels/patch_context.wgsl +29 -0
- package/src/gpu/kernels/pipeline-cache.js +3 -1
- package/src/gpu/kernels/pixel_shuffle.js +1 -1
- package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
- package/src/gpu/kernels/relu.js +1 -1
- package/src/gpu/kernels/repeat_channels.js +1 -1
- package/src/gpu/kernels/residual.js +2 -2
- package/src/gpu/kernels/rmsnorm-pair.js +1 -1
- package/src/gpu/kernels/rmsnorm-qk.js +1 -1
- package/src/gpu/kernels/rmsnorm-stats.js +14 -113
- package/src/gpu/kernels/rmsnorm.js +1 -1
- package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
- package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
- package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
- package/src/gpu/kernels/rope-precompute.js +160 -0
- package/src/gpu/kernels/rope-qk.js +1 -1
- package/src/gpu/kernels/rope.js +2 -2
- package/src/gpu/kernels/rope_precompute.wgsl +74 -0
- package/src/gpu/kernels/sample.js +5 -1
- package/src/gpu/kernels/scale.js +1 -1
- package/src/gpu/kernels/shader-cache.js +8 -4
- package/src/gpu/kernels/shader-source-scope.js +68 -0
- package/src/gpu/kernels/silu.js +2 -1
- package/src/gpu/kernels/soft-embedding.js +3 -1
- package/src/gpu/kernels/softmax.js +4 -2
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
- package/src/gpu/kernels/split_qg.js +1 -1
- package/src/gpu/kernels/split_qkv.d.ts +2 -0
- package/src/gpu/kernels/split_qkv.js +18 -3
- package/src/gpu/kernels/split_qkv.wgsl +5 -2
- package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
- package/src/gpu/kernels/submit_probe.wgsl +6 -0
- package/src/gpu/kernels/transpose.js +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +4 -535
- package/src/gpu/kernels/uniform-utils.js +7 -2
- package/src/gpu/kernels/upsample2d.js +1 -1
- package/src/gpu/kernels/utils.js +9 -330
- package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
- package/src/gpu/kernels/vision-average-pool.js +68 -0
- package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
- package/src/gpu/kernels/vision-patch-embed.js +117 -0
- package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
- package/src/gpu/kernels/vision-position-embedding.js +79 -0
- package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
- package/src/gpu/kernels/vision-rope-2d.js +73 -0
- package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
- package/src/gpu/kernels/vision-spatial-merge.js +100 -0
- package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
- package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
- package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
- package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
- package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
- package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
- package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
- package/src/gpu/readback-buffer.js +20 -0
- package/src/gpu/submit-probe.js +2 -26
- package/src/gpu/weight-buffer.d.ts +1 -0
- package/src/gpu/weight-buffer.js +7 -0
- package/src/index.d.ts +10 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness/benchmark-execution.js +872 -0
- package/src/inference/browser-harness/benchmark-rerank.js +284 -0
- package/src/inference/browser-harness/decode-diagnostics.js +241 -0
- package/src/inference/browser-harness/diffusion-performance.js +102 -0
- package/src/inference/browser-harness/inference-execution.js +543 -0
- package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
- package/src/inference/browser-harness/model-resolution.js +137 -0
- package/src/inference/browser-harness/report.js +338 -0
- package/src/inference/browser-harness/request.js +263 -0
- package/src/inference/browser-harness/runtime-config.d.ts +25 -0
- package/src/inference/browser-harness/runtime-config.js +255 -0
- package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
- package/src/inference/browser-harness/runtime-isolation.js +45 -0
- package/src/inference/browser-harness/runtime-report.d.ts +26 -0
- package/src/inference/browser-harness/runtime-report.js +117 -0
- package/src/inference/browser-harness/suite-summary.js +39 -0
- package/src/inference/browser-harness/text-evidence.js +694 -0
- package/src/inference/browser-harness/text-execution.js +757 -0
- package/src/inference/browser-harness/text-input.js +725 -0
- package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
- package/src/inference/browser-harness-contract-helpers.js +1 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
- package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
- package/src/inference/browser-harness-runtime-helpers.js +3 -423
- package/src/inference/browser-harness-text-helpers.js +31 -2199
- package/src/inference/browser-harness.d.ts +15 -2
- package/src/inference/browser-harness.js +38 -1952
- package/src/inference/kv-cache/base.js +67 -381
- package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
- package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
- package/src/inference/kv-cache/quantized.d.ts +2 -2
- package/src/inference/kv-cache/quantized.js +4 -8
- package/src/inference/kv-cache/update.js +309 -0
- package/src/inference/model-checkpoint-evidence.d.ts +75 -0
- package/src/inference/model-checkpoint-evidence.js +264 -0
- package/src/inference/moe-router.js +36 -216
- package/src/inference/multi-model-network.js +5 -341
- package/src/inference/pipelines/audio/index.js +1 -20
- package/src/inference/pipelines/audio/mel.js +0 -38
- package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
- package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
- package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
- package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
- package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
- package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
- package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
- package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
- package/src/inference/pipelines/factory.js +14 -3
- package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +0 -6
- package/src/inference/pipelines/text/abort-contract.js +22 -0
- package/src/inference/pipelines/text/attention/attn-config.js +1 -16
- package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
- package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
- package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
- package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
- package/src/inference/pipelines/text/attention/interpreter.js +89 -439
- package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
- package/src/inference/pipelines/text/attention/plan.js +17 -0
- package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
- package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
- package/src/inference/pipelines/text/attention/projections.js +20 -537
- package/src/inference/pipelines/text/attention/query-transform.js +11 -0
- package/src/inference/pipelines/text/attention/record.js +0 -6
- package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
- package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
- package/src/inference/pipelines/text/attention/run.js +0 -6
- package/src/inference/pipelines/text/attention/types.js +3 -31
- package/src/inference/pipelines/text/chat-format.d.ts +3 -0
- package/src/inference/pipelines/text/chat-format.js +42 -0
- package/src/inference/pipelines/text/config/normalization.js +729 -0
- package/src/inference/pipelines/text/config/validation.js +9 -0
- package/src/inference/pipelines/text/config.d.ts +12 -1
- package/src/inference/pipelines/text/config.js +14 -1228
- package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
- package/src/inference/pipelines/text/debug-utils/index.js +3 -8
- package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
- package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
- package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +1 -68
- package/src/inference/pipelines/text/embed.js +178 -241
- package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
- package/src/inference/pipelines/text/embedding-contract.js +21 -0
- package/src/inference/pipelines/text/embedding-extraction.js +225 -0
- package/src/inference/pipelines/text/embedding-normalization.js +39 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
- package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
- package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
- package/src/inference/pipelines/text/execution-v1.js +7 -283
- package/src/inference/pipelines/text/execution.js +517 -0
- package/src/inference/pipelines/text/ffn/dense-executor.js +718 -0
- package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
- package/src/inference/pipelines/text/ffn/dense.js +4 -968
- package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
- package/src/inference/pipelines/text/ffn/standard.js +2 -1
- package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
- package/src/inference/pipelines/text/generator/decode-runtime.js +991 -0
- package/src/inference/pipelines/text/generator/decode.js +952 -0
- package/src/inference/pipelines/text/generator/diffusion.js +607 -0
- package/src/inference/pipelines/text/generator/logits-config.js +63 -0
- package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
- package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
- package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
- package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
- package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
- package/src/inference/pipelines/text/generator/recovery.js +310 -0
- package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
- package/src/inference/pipelines/text/generator/text.js +448 -0
- package/src/inference/pipelines/text/generator/training.js +752 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
- package/src/inference/pipelines/text/generator-logits-observation.js +45 -0
- package/src/inference/pipelines/text/generator-runtime.js +5 -135
- package/src/inference/pipelines/text/generator-steps.js +30 -972
- package/src/inference/pipelines/text/generator.js +165 -3544
- package/src/inference/pipelines/text/image-transcription.js +304 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
- package/src/inference/pipelines/text/init-rope.js +37 -189
- package/src/inference/pipelines/text/init.d.ts +8 -3
- package/src/inference/pipelines/text/init.js +6 -216
- package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
- package/src/inference/pipelines/text/layer-input-execution.js +435 -0
- package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
- package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
- package/src/inference/pipelines/text/layer-plan.js +45 -3
- package/src/inference/pipelines/text/layer.js +52 -522
- package/src/inference/pipelines/text/lifecycle.js +839 -0
- package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
- package/src/inference/pipelines/text/linear-attention.js +7 -537
- package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
- package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
- package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
- package/src/inference/pipelines/text/logits/cpu.js +8 -95
- package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
- package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
- package/src/inference/pipelines/text/logits/gpu.js +116 -579
- package/src/inference/pipelines/text/logits/index.d.ts +2 -1
- package/src/inference/pipelines/text/logits/index.js +62 -102
- package/src/inference/pipelines/text/logits/output-transform.js +47 -0
- package/src/inference/pipelines/text/logits/plan.js +184 -0
- package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
- package/src/inference/pipelines/text/logits/readback.js +9 -0
- package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/lora-apply.js +12 -4
- package/src/inference/pipelines/text/lora-types.d.ts +12 -0
- package/src/inference/pipelines/text/modality-token-contract.js +115 -0
- package/src/inference/pipelines/text/model-load.js +0 -20
- package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
- package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
- package/src/inference/pipelines/text/moe/plan.js +47 -0
- package/src/inference/pipelines/text/moe-cpu.js +9 -135
- package/src/inference/pipelines/text/moe-gpu.js +21 -545
- package/src/inference/pipelines/text/normalization-contract.js +17 -0
- package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
- package/src/inference/pipelines/text/per-layer/plan.js +362 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
- package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
- package/src/inference/pipelines/text/probes.js +20 -0
- package/src/inference/pipelines/text/state.d.ts +8 -0
- package/src/inference/pipelines/text/state.js +1 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
- package/src/inference/pipelines/text/types.d.ts +4 -0
- package/src/inference/pipelines/text/weight-loading-config.js +194 -0
- package/src/inference/pipelines/text.d.ts +2 -2
- package/src/inference/pipelines/text.js +46 -1633
- package/src/inference/pipelines/video/frame-sampler.js +0 -13
- package/src/inference/pipelines/video/index.js +0 -21
- package/src/inference/pipelines/vision/attention.js +59 -0
- package/src/inference/pipelines/vision/encoder.js +254 -369
- package/src/inference/pipelines/vision/gemma4.js +204 -318
- package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
- package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
- package/src/inference/pipelines/vision/glmocr.js +557 -0
- package/src/inference/pipelines/vision/image-preprocess.js +0 -27
- package/src/inference/pipelines/vision/index.js +8 -27
- package/src/inference/pipelines/vision/ops.js +11 -33
- package/src/inference/pipelines/vision/patch-embed.js +51 -145
- package/src/inference/pipelines/vision/spatial-merge.js +78 -0
- package/src/inference/pipelines/vision/weight-loading.js +79 -0
- package/src/inference/refactor-receipt.js +2 -2
- package/src/inference/rerank.js +182 -0
- package/src/inference/tokenizer.js +1 -1
- package/src/inference/tokenizers/behavior-flags.js +6 -3
- package/src/inference/tokenizers/bundled/decoders.js +133 -0
- package/src/inference/tokenizers/bundled/registry.js +216 -0
- package/src/inference/tokenizers/bundled.js +23 -353
- package/src/loader/doppler-loader.d.ts +4 -0
- package/src/loader/doppler-loader.js +119 -930
- package/src/loader/embedding-limit-preflight.js +11 -4
- package/src/loader/embedding-loader.js +1 -3
- package/src/loader/gpu-tensor-materialization.js +20 -0
- package/src/loader/layer-loader.js +15 -119
- package/src/loader/layer-tensor-aliases.js +99 -0
- package/src/loader/layer-weight-shape.js +29 -0
- package/src/loader/load-timing.js +64 -0
- package/src/loader/loader-types.d.ts +3 -0
- package/src/loader/model-executor.js +819 -0
- package/src/loader/model-load-validation.js +15 -0
- package/src/loader/quantization-constants.d.ts +11 -29
- package/src/loader/quantization-constants.js +1 -2
- package/src/loader/shard-resolver.js +0 -14
- package/src/loader/tensor-load-error.js +14 -0
- package/src/loader/tensor-shard-indices.js +8 -0
- package/src/loader/tensors/bf16-loader.js +125 -0
- package/src/loader/tensors/functional-descriptor-loader.js +3 -375
- package/src/loader/tensors/load-executor.js +577 -0
- package/src/loader/tensors/source-transform/executor.js +123 -0
- package/src/loader/tensors/source-transform/plan.js +107 -0
- package/src/loader/tensors/source-transform.js +15 -264
- package/src/loader/tensors/tensor-loader.d.ts +4 -4
- package/src/loader/tensors/tensor-loader.js +7 -653
- package/src/memory/buffer-pool.js +7 -29
- package/src/rules/capability-transform-resolver.js +80 -0
- package/src/rules/execution-contract.d.ts +73 -0
- package/src/rules/execution-contract.js +429 -0
- package/src/rules/execution-rules-contract-check.js +1 -1
- package/src/rules/generated/rule-bundle.json +4305 -0
- package/src/rules/inference/capability-transforms.rules.json +3 -3
- package/src/rules/layer-pattern-contract-check.js +1 -1
- package/src/rules/rule-registry.js +70 -57
- package/src/storage/artifact-storage-context.js +2 -2
- package/src/storage/backend-selection.js +43 -0
- package/src/storage/distribution-transport.js +5 -0
- package/src/storage/download/integrity.d.ts +33 -0
- package/src/storage/download/integrity.js +130 -0
- package/src/storage/download/resume-inspection.d.ts +22 -0
- package/src/storage/download/resume-inspection.js +134 -0
- package/src/storage/download/retry.js +57 -0
- package/src/storage/download/state.js +131 -0
- package/src/storage/download/transport.js +99 -0
- package/src/storage/downloader.js +74 -453
- package/src/storage/model-cache.d.ts +1 -0
- package/src/storage/model-cache.js +51 -11
- package/src/storage/shard-manager.d.ts +5 -0
- package/src/storage/shard-manager.js +50 -369
- package/src/storage/shards/index.js +60 -0
- package/src/storage/shards/integrity.js +225 -0
- package/src/storage/shards/lifecycle.js +22 -0
- package/src/storage/source-artifact-store.js +14 -4
- package/src/tooling/boundary-evidence.d.ts +4 -4
- package/src/tooling/boundary-evidence.js +20 -20
- package/src/tooling/browser-command-runner.js +13 -6
- package/src/tooling/capsule-v2.d.ts +11 -0
- package/src/tooling/capsule-v2.js +58 -0
- package/src/tooling/capsule.js +20 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.js +163 -5
- package/src/tooling/command-api.d.ts +42 -3
- package/src/tooling/command-api.js +7 -2
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
- package/src/tooling/command-runner.html +49 -1
- package/src/tooling/command-validation.js +1 -24
- package/src/tooling/conversion-run-timing.js +24 -0
- package/src/tooling/diagnose-runner.js +11 -9
- package/src/tooling/distribution-shard-transport.js +11 -0
- package/src/tooling/execution-cost-ledger.d.ts +76 -2
- package/src/tooling/execution-cost-ledger.js +197 -15
- package/src/tooling/hf-registry/catalog-io.js +63 -0
- package/src/tooling/litert/package.js +863 -0
- package/src/tooling/litert-package-runtime.js +3 -849
- package/src/tooling/model-capsule-forge.js +478 -0
- package/src/tooling/node-browser/execution.d.ts +40 -0
- package/src/tooling/node-browser/execution.js +308 -0
- package/src/tooling/node-browser/static-server.js +188 -0
- package/src/tooling/node-browser/transport.d.ts +80 -0
- package/src/tooling/node-browser/transport.js +622 -0
- package/src/tooling/node-browser-command-runner.d.ts +18 -78
- package/src/tooling/node-browser-command-runner.js +10 -1112
- package/src/tooling/node-capsule-artifact-store.js +44 -0
- package/src/tooling/node-command-execution.js +237 -0
- package/src/tooling/node-command-runner.js +26 -205
- package/src/tooling/node-converter/input.js +521 -0
- package/src/tooling/node-converter/output.js +244 -0
- package/src/tooling/node-converter-chunk-batches.js +21 -0
- package/src/tooling/node-converter.js +7 -720
- package/src/tooling/node-source-runtime.js +16 -276
- package/src/tooling/node-webgpu-provider.v1.json +33 -0
- package/src/tooling/node-webgpu.js +178 -453
- package/src/tooling/opfs-cache.d.ts +5 -0
- package/src/tooling/opfs-cache.js +4 -0
- package/src/tooling/production-release.d.ts +19 -0
- package/src/tooling/production-release.js +647 -0
- package/src/tooling/program-bundle/embedding-reference.js +19 -0
- package/src/tooling/program-bundle/materialize.js +324 -0
- package/src/tooling/program-bundle/rerank-reference.js +22 -0
- package/src/tooling/program-bundle/sequence-reference.js +42 -0
- package/src/tooling/program-bundle/validation.js +31 -0
- package/src/tooling/program-bundle/wgsl-closure.js +254 -0
- package/src/tooling/program-bundle-parity.d.ts +11 -4
- package/src/tooling/program-bundle-parity.js +262 -109
- package/src/tooling/program-bundle.d.ts +7 -0
- package/src/tooling/program-bundle.js +186 -591
- package/src/tooling/provider-v1-contract.d.ts +61 -0
- package/src/tooling/provider-v1-contract.js +192 -0
- package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
- package/src/tooling/registered-variant-calibration-job.js +210 -0
- package/src/tooling/registered-variant-calibration.d.ts +167 -7
- package/src/tooling/registered-variant-calibration.js +57 -7
- package/src/tooling/runtime-optimization/candidates.js +664 -0
- package/src/tooling/runtime-optimization-campaign.js +166 -0
- package/src/tooling/runtime-optimization-index.js +16 -1
- package/src/tooling/runtime-optimization.d.ts +44 -0
- package/src/tooling/runtime-optimization.js +13 -656
- package/src/tooling/source-intake.d.ts +1 -1
- package/src/tooling/source-intake.js +34 -17
- package/src/tooling/source-package-profiles.js +1 -1
- package/src/tooling/source-runtime/bundle-materialize.js +298 -0
- package/src/tooling/source-runtime/bundle-plan.js +121 -0
- package/src/tooling/source-runtime/node-loader.js +258 -0
- package/src/tooling/source-runtime-bundle.js +26 -411
- package/src/tooling/source-runtime-converter-config.js +1 -1
- package/src/tooling-experimental-exports.js +0 -5
- package/src/tooling-exports/evidence.d.ts +14 -3
- package/src/tooling-exports/evidence.js +14 -3
- package/src/tooling-exports/storage.d.ts +5 -0
- package/src/tooling-exports/storage.js +4 -0
- package/src/tooling-exports.d.ts +3 -0
- package/src/tooling-exports.js +2 -0
- package/src/tooling-exports.shared.d.ts +17 -3
- package/src/tooling-exports.shared.js +9 -3
- package/src/training.d.ts +6 -6
- package/src/training.js +4 -4
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +512 -0
- package/tests/kernels/browser/test-page.js +169 -14
- package/tests/kernels/harness/tolerance.js +1 -0
- package/tests/kernels/reference/index.js +13 -1
- package/tests/kernels/reference/vision.js +160 -0
- package/src/converter/quantizer.d.ts +0 -121
- package/src/gpu/kernels/backward/utils.d.ts +0 -90
- package/src/gpu/kernels/uniform-utils.d.ts +0 -66
- package/src/gpu/kernels/utils.d.ts +0 -147
- package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
- package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
- package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
- package/src/inference/pipelines/text/moe-helpers.js +0 -21
- package/src/rules/converter/execution.rules.json +0 -20
- package/src/rules/converter/tensor-roles.rules.json +0 -13
- package/src/rules/converter/tokenizer.rules.json +0 -12
- package/src/rules/inference/attention.rules.json +0 -69
- package/src/rules/inference/config.rules.json +0 -82
- package/src/rules/inference/dtype.rules.json +0 -113
- package/src/rules/inference/execution.rules.json +0 -195
- package/src/rules/inference/ffn.rules.json +0 -123
- package/src/rules/inference/layer-pattern.rules.json +0 -16
- package/src/rules/inference/layer.rules.json +0 -7
- package/src/rules/inference/moe.rules.json +0 -145
- package/src/rules/kernels/attention.rules.json +0 -68
- package/src/rules/kernels/conv2d.rules.json +0 -6
- package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
- package/src/rules/kernels/dequant.rules.json +0 -58
- package/src/rules/kernels/energy.rules.json +0 -22
- package/src/rules/kernels/fused-ffn.rules.json +0 -15
- package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
- package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
- package/src/rules/kernels/gather.rules.json +0 -93
- package/src/rules/kernels/gelu.rules.json +0 -11
- package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
- package/src/rules/kernels/groupnorm.rules.json +0 -10
- package/src/rules/kernels/kv_quantize.rules.json +0 -9
- package/src/rules/kernels/layernorm.rules.json +0 -6
- package/src/rules/kernels/linear-attention.rules.json +0 -6
- package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
- package/src/rules/kernels/matmul.rules.json +0 -95
- package/src/rules/kernels/modulate.rules.json +0 -6
- package/src/rules/kernels/moe.rules.gptoss.json +0 -105
- package/src/rules/kernels/moe.rules.json +0 -14
- package/src/rules/kernels/moe.rules.mixtral.json +0 -75
- package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
- package/src/rules/kernels/relu.rules.json +0 -6
- package/src/rules/kernels/rep-penalty.rules.json +0 -6
- package/src/rules/kernels/repeat-channels.rules.json +0 -6
- package/src/rules/kernels/residual.rules.json +0 -12
- package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
- package/src/rules/kernels/rmsnorm.rules.json +0 -46
- package/src/rules/kernels/rope-qk.rules.json +0 -6
- package/src/rules/kernels/rope.rules.json +0 -6
- package/src/rules/kernels/sample.rules.json +0 -6
- package/src/rules/kernels/scale.rules.json +0 -8
- package/src/rules/kernels/silu.rules.json +0 -22
- package/src/rules/kernels/softmax.rules.json +0 -25
- package/src/rules/kernels/split-qg.rules.json +0 -6
- package/src/rules/kernels/split-qkv.rules.json +0 -6
- package/src/rules/kernels/upsample2d.rules.json +0 -6
- package/src/rules/loader/tensor-loader.rules.json +0 -42
- package/src/rules/loader/weights.rules.json +0 -42
- package/src/rules/tooling/command-runtime.rules.json +0 -65
- package/src/tooling/hf-registry-utils.js +0 -395
- /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
- /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
- /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
- /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
- /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
- /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
- /package/src/{utils → formats}/array-buffer.js +0 -0
- /package/src/{utils → formats}/canonical-hash.js +0 -0
- /package/src/{utils → formats}/clone-json.js +0 -0
- /package/src/{utils → formats}/encode-utf8.js +0 -0
- /package/src/{utils → formats}/load-json.js +0 -0
- /package/src/{utils → formats}/plain-object.js +0 -0
- /package/src/{utils → formats}/sha256.js +0 -0
- /package/src/{utils → formats}/stable-sort-object.js +0 -0
- /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
- /package/src/{utils → storage}/hf-resolve-url.js +0 -0
- /package/src/{utils → storage}/runtime-env.js +0 -0
- /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
|
@@ -48,11 +48,13 @@ import {
|
|
|
48
48
|
import {
|
|
49
49
|
buildLayerContext,
|
|
50
50
|
debugCheckBuffer as debugCheckBufferHelper,
|
|
51
|
+
resolvePerLayerInputsSession,
|
|
52
|
+
} from './generator/session-context.js';
|
|
53
|
+
import {
|
|
51
54
|
getLogitsConfig,
|
|
52
55
|
getLogitsWeights,
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
} from './generator-helpers.js';
|
|
56
|
+
} from './generator/logits-config.js';
|
|
57
|
+
import { releaseSharedAttentionState } from './generator/attention-lifecycle.js';
|
|
56
58
|
import {
|
|
57
59
|
assertTokenIdsInRange,
|
|
58
60
|
assertTokenIdInRange,
|
|
@@ -70,7 +72,11 @@ import { resolveSamplingConfig } from './sampling-config.js';
|
|
|
70
72
|
import { decodeReadback, getLogitsHealth } from './debug-utils/index.js';
|
|
71
73
|
import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
|
|
72
74
|
import { resolveDeferredRoundingWindowTokens } from './finiteness-policy.js';
|
|
73
|
-
import {
|
|
75
|
+
import {
|
|
76
|
+
advanceDecodeStepCount,
|
|
77
|
+
createTsirFixtureState,
|
|
78
|
+
drainPendingTsirReads,
|
|
79
|
+
} from './tsir-fixture-writer.js';
|
|
74
80
|
import {
|
|
75
81
|
activateFallbackExecutionPlan,
|
|
76
82
|
hasFallbackExecutionPlan,
|
|
@@ -105,15 +111,19 @@ import {
|
|
|
105
111
|
// Extracted standalone helpers
|
|
106
112
|
import {
|
|
107
113
|
resolvePromptInput,
|
|
114
|
+
} from './generator/prompt-input.js';
|
|
115
|
+
import {
|
|
108
116
|
releasePerLayerInputBuffer,
|
|
117
|
+
shouldDisablePrefillCommandBatching,
|
|
118
|
+
resolveEffectivePrefillTokenChunkSize,
|
|
119
|
+
} from './generator/prefill-policy.js';
|
|
120
|
+
import {
|
|
109
121
|
normalizePrefixEmbeddingOverride,
|
|
110
122
|
resolvePrefillEmbeddingInputIds,
|
|
111
123
|
resolvePrefillMultimodalBidirectionalSpan,
|
|
112
124
|
applyPrefixEmbeddingOverride,
|
|
113
125
|
resolvePrefixEmbeddingOverrideTransitionDeclaredBy,
|
|
114
|
-
|
|
115
|
-
resolveEffectivePrefillTokenChunkSize,
|
|
116
|
-
} from './generator-prefill-helpers.js';
|
|
126
|
+
} from './generator/prefix-embedding.js';
|
|
117
127
|
import {
|
|
118
128
|
shouldDisableBatchDecodeAfterShortBatch,
|
|
119
129
|
resolveHotVocabularyBatchDecodeAvailability,
|
|
@@ -129,360 +139,94 @@ import {
|
|
|
129
139
|
resolveCurrentKVCacheDtype,
|
|
130
140
|
cloneRuntimeInferenceWithKVDtype,
|
|
131
141
|
} from './generator-decode-policy.js';
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
`[Pipeline] runtime.inference.session.prefillChunkSubmitMode must be "sync" or "async"; got "${String(submit)}".`
|
|
180
|
-
);
|
|
181
|
-
}
|
|
182
|
-
return submit;
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
function getTokenizerSuppressionCache(tokenizer) {
|
|
186
|
-
let cache = tokenizerSuppressionCache.get(tokenizer);
|
|
187
|
-
if (!cache) {
|
|
188
|
-
cache = new Map();
|
|
189
|
-
tokenizerSuppressionCache.set(tokenizer, cache);
|
|
190
|
-
}
|
|
191
|
-
return cache;
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
function decodeSingleTokenForSuppression(tokenizer, tokenId) {
|
|
195
|
-
if (!tokenizer || typeof tokenizer.decode !== 'function') {
|
|
196
|
-
return '';
|
|
197
|
-
}
|
|
198
|
-
try {
|
|
199
|
-
return tokenizer.decode([tokenId], false, false);
|
|
200
|
-
} catch {
|
|
201
|
-
return '';
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
function collectTokenizerSuppressedTokenIds(tokenizer, vocabSize, samplingConfig) {
|
|
206
|
-
if (!tokenizer || !Number.isInteger(vocabSize) || vocabSize < 1) {
|
|
207
|
-
return [];
|
|
208
|
-
}
|
|
209
|
-
if (samplingConfig.suppressSpecialTokens !== true && samplingConfig.suppressSpecialLikeTokens !== true) {
|
|
210
|
-
return [];
|
|
211
|
-
}
|
|
212
|
-
const cache = getTokenizerSuppressionCache(tokenizer);
|
|
213
|
-
const cacheKey = [
|
|
214
|
-
vocabSize,
|
|
215
|
-
samplingConfig.suppressSpecialTokens === true ? 'special' : 'plain',
|
|
216
|
-
samplingConfig.suppressSpecialLikeTokens === true ? 'specialLike' : 'literal',
|
|
217
|
-
].join(':');
|
|
218
|
-
const cached = cache.get(cacheKey);
|
|
219
|
-
if (cached) {
|
|
220
|
-
return cached;
|
|
221
|
-
}
|
|
222
|
-
const tokenIds = [];
|
|
223
|
-
for (let tokenId = 0; tokenId < vocabSize; tokenId++) {
|
|
224
|
-
const tokenizerSpecial = samplingConfig.suppressSpecialTokens === true
|
|
225
|
-
&& typeof tokenizer.isSpecialToken === 'function'
|
|
226
|
-
&& tokenizer.isSpecialToken(tokenId);
|
|
227
|
-
const specialLike = samplingConfig.suppressSpecialLikeTokens === true
|
|
228
|
-
&& SPECIAL_LIKE_TOKEN_RE.test(decodeSingleTokenForSuppression(tokenizer, tokenId).trim());
|
|
229
|
-
if (tokenizerSpecial || specialLike) {
|
|
230
|
-
tokenIds.push(tokenId);
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
cache.set(cacheKey, tokenIds);
|
|
234
|
-
return tokenIds;
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
function resolveSuppressedSamplingTokenIds(state, samplingConfig) {
|
|
238
|
-
const vocabSize = state.modelConfig?.vocabSize;
|
|
239
|
-
const suppressed = new Set(samplingConfig.suppressTokenIds);
|
|
240
|
-
const stopTokenIds = new Set(state.modelConfig?.stopTokenIds ?? []);
|
|
241
|
-
const eosToken = state.tokenizer?.getSpecialTokens?.()?.eos;
|
|
242
|
-
if (Number.isInteger(eosToken)) {
|
|
243
|
-
stopTokenIds.add(eosToken);
|
|
244
|
-
}
|
|
245
|
-
for (const tokenId of collectTokenizerSuppressedTokenIds(state.tokenizer, vocabSize, samplingConfig)) {
|
|
246
|
-
if (!stopTokenIds.has(tokenId)) {
|
|
247
|
-
suppressed.add(tokenId);
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
return [...suppressed];
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
function normalizeSelectedLogitTokenIds(value, vocabSize, label) {
|
|
254
|
-
if (value == null) {
|
|
255
|
-
return null;
|
|
256
|
-
}
|
|
257
|
-
if (!Array.isArray(value) && !ArrayBuffer.isView(value)) {
|
|
258
|
-
throw new Error(`${label} must be an array or typed array.`);
|
|
259
|
-
}
|
|
260
|
-
const tokenIds = Array.from(value, (entry, index) => {
|
|
261
|
-
const tokenId = Number(entry);
|
|
262
|
-
if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSize) {
|
|
263
|
-
throw new Error(`${label}[${index}] must be an integer in [0, ${vocabSize}), got "${String(entry)}".`);
|
|
264
|
-
}
|
|
265
|
-
return tokenId;
|
|
266
|
-
});
|
|
267
|
-
if (tokenIds.length === 0) {
|
|
268
|
-
throw new Error(`${label} must not be empty.`);
|
|
269
|
-
}
|
|
270
|
-
return tokenIds;
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
async function traceActivationHealth(label, buffer, dtype, elementCount) {
|
|
274
|
-
if (!isTraceEnabled('logits') || !isGpuBufferInstance(buffer)) {
|
|
275
|
-
return;
|
|
276
|
-
}
|
|
277
|
-
const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
|
|
278
|
-
const data = await readBuffer(buffer, elementCount * bytesPerElement);
|
|
279
|
-
trace.logits(label, getLogitsHealth(decodeReadback(data, dtype)));
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
function ownsBorrowedWeightBuffer(weight) {
|
|
283
|
-
return !isGpuBufferInstance(weight) && !isWeightBuffer(weight);
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
function borrowLinearWeight(weight, label) {
|
|
287
|
-
if (!weight) {
|
|
288
|
-
throw new Error(`DiffusionGemma self-conditioning missing ${label}.`);
|
|
289
|
-
}
|
|
290
|
-
if (isSplitWeightBuffer(weight)) {
|
|
291
|
-
throw new Error(
|
|
292
|
-
`DiffusionGemma self-conditioning does not support split weight storage for ${label}.`
|
|
293
|
-
);
|
|
294
|
-
}
|
|
295
|
-
return {
|
|
296
|
-
value: getPipelineWeightBuffer(weight, label),
|
|
297
|
-
owned: ownsBorrowedWeightBuffer(weight),
|
|
298
|
-
};
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
function borrowNormWeight(weight, label) {
|
|
302
|
-
if (!weight) {
|
|
303
|
-
throw new Error(`DiffusionGemma self-conditioning missing ${label}.`);
|
|
304
|
-
}
|
|
305
|
-
if (isSplitWeightBuffer(weight)) {
|
|
306
|
-
throw new Error(
|
|
307
|
-
`DiffusionGemma self-conditioning does not support split norm storage for ${label}.`
|
|
308
|
-
);
|
|
309
|
-
}
|
|
310
|
-
return {
|
|
311
|
-
value: getPipelineNormWeightBuffer(weight, label),
|
|
312
|
-
owned: ownsBorrowedWeightBuffer(weight),
|
|
313
|
-
};
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
function releaseBorrowedWeight(borrowed) {
|
|
317
|
-
if (!borrowed?.owned) {
|
|
318
|
-
return;
|
|
319
|
-
}
|
|
320
|
-
const value = borrowed.value;
|
|
321
|
-
releaseBuffer(isWeightBuffer(value) ? value.buffer : value);
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
function canUseChunkedSoftEmbeddingLogits(logitsState, embeddingWeight, embeddingTranspose) {
|
|
325
|
-
return logitsState != null
|
|
326
|
-
&& isWeightBuffer(embeddingWeight)
|
|
327
|
-
&& getWeightDtype(embeddingWeight) === 'f16'
|
|
328
|
-
&& getLayout(embeddingWeight) === 'row'
|
|
329
|
-
&& embeddingTranspose !== true;
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
function resolveDiffusionGemmaSoftEmbeddingChunkRows(runtimeConfig) {
|
|
333
|
-
const value = runtimeConfig?.inference?.diffusionGemma?.softEmbeddingLogitsChunkRows;
|
|
334
|
-
if (value == null) {
|
|
335
|
-
return undefined;
|
|
336
|
-
}
|
|
337
|
-
if (!Number.isInteger(value) || value <= 0) {
|
|
338
|
-
throw new Error(
|
|
339
|
-
'runtime.inference.diffusionGemma.softEmbeddingLogitsChunkRows must be a positive integer.'
|
|
340
|
-
);
|
|
341
|
-
}
|
|
342
|
-
return value;
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
function normalizeCanvasTokenIds(canvas, context) {
|
|
346
|
-
if (!Array.isArray(canvas) && !ArrayBuffer.isView(canvas)) {
|
|
347
|
-
throw new Error(`[DiffusionGemma] ${context}.canvas must be an array or typed array of token IDs.`);
|
|
348
|
-
}
|
|
349
|
-
return Array.from(canvas, (value, index) => {
|
|
350
|
-
if (!Number.isFinite(value) || Math.floor(value) !== value || value < 0) {
|
|
351
|
-
throw new Error(`[DiffusionGemma] ${context}.canvas[${index}] must be a non-negative integer token ID.`);
|
|
352
|
-
}
|
|
353
|
-
return value;
|
|
354
|
-
});
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
function normalizeSelfConditioningLogits(logits, canvasLength, vocabSize) {
|
|
358
|
-
if (logits == null) {
|
|
359
|
-
return null;
|
|
360
|
-
}
|
|
361
|
-
const expected = canvasLength * vocabSize;
|
|
362
|
-
const values = logits instanceof Float32Array
|
|
363
|
-
? logits
|
|
364
|
-
: (Array.isArray(logits) ? Float32Array.from(logits) : null);
|
|
365
|
-
if (!values) {
|
|
366
|
-
throw new Error('[DiffusionGemma] selfConditioningLogits must be a Float32Array or number array.');
|
|
367
|
-
}
|
|
368
|
-
if (values.length !== expected) {
|
|
369
|
-
throw new Error(
|
|
370
|
-
`[DiffusionGemma] selfConditioningLogits length mismatch: expected ${expected}, got ${values.length}.`
|
|
371
|
-
);
|
|
372
|
-
}
|
|
373
|
-
return values;
|
|
374
|
-
}
|
|
375
|
-
|
|
376
|
-
function normalizeSelfConditioningLogitsState(logits, canvasLength, vocabSize) {
|
|
377
|
-
if (logits == null || !isGpuBufferInstance(logits.logitsBuffer)) {
|
|
378
|
-
return null;
|
|
379
|
-
}
|
|
380
|
-
const dtype = logits.logitsDtype;
|
|
381
|
-
if (dtype !== 'f32') {
|
|
382
|
-
throw new Error(`[DiffusionGemma] GPU selfConditioningLogits require f32 logits, got "${dtype}".`);
|
|
383
|
-
}
|
|
384
|
-
if (logits.vocabSize !== vocabSize) {
|
|
385
|
-
throw new Error(
|
|
386
|
-
`[DiffusionGemma] GPU selfConditioningLogits vocab mismatch: expected ${vocabSize}, got ${logits.vocabSize}.`
|
|
387
|
-
);
|
|
388
|
-
}
|
|
389
|
-
if (logits.canvasLength !== canvasLength) {
|
|
390
|
-
throw new Error(
|
|
391
|
-
`[DiffusionGemma] GPU selfConditioningLogits canvas mismatch: expected ${canvasLength}, got ${logits.canvasLength}.`
|
|
392
|
-
);
|
|
393
|
-
}
|
|
394
|
-
const temperature = logits.temperature ?? 1.0;
|
|
395
|
-
if (typeof temperature !== 'number' || !Number.isFinite(temperature) || temperature <= 0) {
|
|
396
|
-
throw new Error('[DiffusionGemma] GPU selfConditioningLogits temperature must be positive.');
|
|
397
|
-
}
|
|
398
|
-
return {
|
|
399
|
-
logitsBuffer: logits.logitsBuffer,
|
|
400
|
-
logitsDtype: dtype,
|
|
401
|
-
temperature,
|
|
402
|
-
releaseOnUse: logits.releaseOnUse !== false,
|
|
403
|
-
};
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
function normalizeSelfConditioningSoftEmbeddingState(state, canvasLength, hiddenSize) {
|
|
407
|
-
if (state?.kind !== 'soft_embedding') {
|
|
408
|
-
return null;
|
|
409
|
-
}
|
|
410
|
-
if (!isGpuBufferInstance(state.buffer)) {
|
|
411
|
-
throw new Error('[DiffusionGemma] GPU selfConditioning soft embedding requires a GPU buffer.');
|
|
412
|
-
}
|
|
413
|
-
const dtype = state.dtype ?? 'f32';
|
|
414
|
-
if (dtype !== 'f32') {
|
|
415
|
-
throw new Error(`[DiffusionGemma] GPU selfConditioning soft embedding requires f32 dtype, got "${dtype}".`);
|
|
416
|
-
}
|
|
417
|
-
if (state.canvasLength !== canvasLength) {
|
|
418
|
-
throw new Error(
|
|
419
|
-
`[DiffusionGemma] GPU selfConditioning soft embedding canvas mismatch: ` +
|
|
420
|
-
`expected ${canvasLength}, got ${state.canvasLength}.`
|
|
421
|
-
);
|
|
422
|
-
}
|
|
423
|
-
if (state.hiddenSize !== hiddenSize) {
|
|
424
|
-
throw new Error(
|
|
425
|
-
`[DiffusionGemma] GPU selfConditioning soft embedding hidden mismatch: ` +
|
|
426
|
-
`expected ${hiddenSize}, got ${state.hiddenSize}.`
|
|
427
|
-
);
|
|
428
|
-
}
|
|
429
|
-
return {
|
|
430
|
-
buffer: state.buffer,
|
|
431
|
-
dtype,
|
|
432
|
-
releaseOnUse: state.releaseOnUse !== false,
|
|
433
|
-
scaled: state.scaled === true,
|
|
434
|
-
};
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
let intentBundleModulePromise = null;
|
|
438
|
-
|
|
439
|
-
async function getExperimentalIntentBundleModule() {
|
|
440
|
-
intentBundleModulePromise ??= import('../../../experimental/hotswap/intent-bundle.js');
|
|
441
|
-
return intentBundleModulePromise;
|
|
442
|
-
}
|
|
443
|
-
|
|
142
|
+
import { FINITENESS_RESET_WORDS, borrowLinearWeight, borrowNormWeight, canUseChunkedSoftEmbeddingLogits, getExperimentalIntentBundleModule, normalizeCanvasTokenIds, normalizeSelectedLogitTokenIds, normalizeSelfConditioningLogits, normalizeSelfConditioningLogitsState, normalizeSelfConditioningSoftEmbeddingState, recordPrefillRecorderStats, releaseBorrowedWeight, resolveDiffusionGemmaSoftEmbeddingChunkRows, resolvePrefillChunkSubmitMode, resolveSuppressedSamplingTokenIds, traceActivationHealth } from './generator/text.js';
|
|
143
|
+
import {
|
|
144
|
+
_createDiffusionGemmaSelfConditioningEmbeddings as _createDiffusionGemmaSelfConditioningEmbeddingsImpl,
|
|
145
|
+
_createDiffusionGemmaSelfConditioningSoftEmbeddingState as _createDiffusionGemmaSelfConditioningSoftEmbeddingStateImpl,
|
|
146
|
+
computeDiffusionGemmaCanvasLogits as computeDiffusionGemmaCanvasLogitsImpl,
|
|
147
|
+
computeDiffusionGemmaCanvasStep as computeDiffusionGemmaCanvasStepImpl,
|
|
148
|
+
} from './generator/diffusion.js';
|
|
149
|
+
import {
|
|
150
|
+
_replayPrefillDecodeLogits as _replayPrefillDecodeLogitsImpl,
|
|
151
|
+
_shouldUseFinitenessFallback as _shouldUseFinitenessFallbackImpl,
|
|
152
|
+
_recreateKVCacheForExecutionPlan as _recreateKVCacheForExecutionPlanImpl,
|
|
153
|
+
_openFinitenessFallbackWindow as _openFinitenessFallbackWindowImpl,
|
|
154
|
+
_closeFinitenessFallbackWindow as _closeFinitenessFallbackWindowImpl,
|
|
155
|
+
_consumeFinitenessFallbackToken as _consumeFinitenessFallbackTokenImpl,
|
|
156
|
+
_beginFinitenessFallback as _beginFinitenessFallbackImpl,
|
|
157
|
+
_endFinitenessFallback as _endFinitenessFallbackImpl,
|
|
158
|
+
_retryWithFinitenessFallback as _retryWithFinitenessFallbackImpl,
|
|
159
|
+
_retryWithPersistentFinitenessFallback as _retryWithPersistentFinitenessFallbackImpl,
|
|
160
|
+
_retryDecodeStepWithFinitenessWindow as _retryDecodeStepWithFinitenessWindowImpl,
|
|
161
|
+
} from './generator/recovery.js';
|
|
162
|
+
import {
|
|
163
|
+
prefillKVOnly as prefillKVOnlyImpl,
|
|
164
|
+
prefillForLoRATraining as prefillForLoRATrainingImpl,
|
|
165
|
+
prefillWithEmbedding as prefillWithEmbeddingImpl,
|
|
166
|
+
prefillWithLogits as prefillWithLogitsImpl,
|
|
167
|
+
prefillWithTokenLogits as prefillWithTokenLogitsImpl,
|
|
168
|
+
prefillWithTokenLogitsFromKV as prefillWithTokenLogitsFromKVImpl,
|
|
169
|
+
generateWithPrefixKV as generateWithPrefixKVImpl,
|
|
170
|
+
} from './generator/training.js';
|
|
171
|
+
import {
|
|
172
|
+
_generateTokensInternal as _generateTokensInternalImpl,
|
|
173
|
+
generateTokenIds as generateTokenIdsImpl,
|
|
174
|
+
_runDecodeLoop as _runDecodeLoopImpl,
|
|
175
|
+
_decodeStep as _decodeStepImpl,
|
|
176
|
+
decodeStepLogits as decodeStepLogitsImpl,
|
|
177
|
+
advanceWithToken as advanceWithTokenImpl,
|
|
178
|
+
advanceWithTokenAndEmbedding as advanceWithTokenAndEmbeddingImpl,
|
|
179
|
+
_generateNTokensGPU as _generateNTokensGPUImpl,
|
|
180
|
+
} from './generator/decode-runtime.js';
|
|
181
|
+
import {
|
|
182
|
+
_commitPrefillHiddenChunk as _commitPrefillHiddenChunkImpl,
|
|
183
|
+
_prefillInputIdsToLogits as _prefillInputIdsToLogitsImpl,
|
|
184
|
+
_prefillPromptToLogits as _prefillPromptToLogitsImpl,
|
|
185
|
+
_prefillToHidden as _prefillToHiddenImpl,
|
|
186
|
+
_prefill as _prefillImpl,
|
|
187
|
+
} from './generator/prefill-runtime.js';
|
|
188
|
+
export { resolvePrefillChunkSubmitMode } from './generator/text.js';
|
|
444
189
|
|
|
445
190
|
export class PipelineGenerator {
|
|
446
191
|
|
|
447
|
-
|
|
448
|
-
|
|
192
|
+
_state;
|
|
193
|
+
_finitenessFallbackWindow;
|
|
449
194
|
|
|
450
195
|
_assertTokenIdsInRange(tokenIds, context = 'encode') {
|
|
451
|
-
assertTokenIdsInRange(this
|
|
196
|
+
assertTokenIdsInRange(this._state, tokenIds, context);
|
|
452
197
|
}
|
|
453
198
|
|
|
454
199
|
_assertTokenIdInRange(tokenId, context = 'token') {
|
|
455
|
-
assertTokenIdInRange(this
|
|
200
|
+
assertTokenIdInRange(this._state, tokenId, context);
|
|
456
201
|
}
|
|
457
202
|
|
|
458
|
-
|
|
459
203
|
constructor(state) {
|
|
460
|
-
this
|
|
461
|
-
this
|
|
204
|
+
this._state = state;
|
|
205
|
+
this._finitenessFallbackWindow = null;
|
|
462
206
|
}
|
|
463
207
|
|
|
464
208
|
_resolveDeferredRoundingWindowTokens() {
|
|
465
|
-
const activePlan = resolveActiveExecutionPlan(this
|
|
209
|
+
const activePlan = resolveActiveExecutionPlan(this._state);
|
|
466
210
|
return activePlan?.deferredRoundingWindowTokens
|
|
467
|
-
?? resolveDeferredRoundingWindowTokens(this
|
|
211
|
+
?? resolveDeferredRoundingWindowTokens(this._state.runtimeConfig?.inference?.compute);
|
|
468
212
|
}
|
|
469
213
|
|
|
470
214
|
_getEffectiveActivationDtype() {
|
|
471
|
-
return resolveActiveExecutionPlan(this
|
|
215
|
+
return resolveActiveExecutionPlan(this._state).activationDtype;
|
|
472
216
|
}
|
|
473
217
|
|
|
474
218
|
_hasFinitenessFallbackWindow() {
|
|
475
|
-
return this
|
|
219
|
+
return this._finitenessFallbackWindow !== null;
|
|
476
220
|
}
|
|
477
221
|
|
|
478
222
|
_resetReplayPrefillRuntimeState() {
|
|
479
|
-
this
|
|
480
|
-
this
|
|
481
|
-
this
|
|
223
|
+
this._state.kvCache?.clear?.();
|
|
224
|
+
this._state.linearAttentionRuntime = resetLinearAttentionRuntime(this._state.linearAttentionRuntime);
|
|
225
|
+
this._state.currentSeqLen = 0;
|
|
482
226
|
}
|
|
483
227
|
|
|
484
228
|
resetGenerationState() {
|
|
485
|
-
if (this
|
|
229
|
+
if (this._state.isGenerating) {
|
|
486
230
|
throw new Error('InferencePipeline.resetGenerationState: cannot reset while generation is in progress');
|
|
487
231
|
}
|
|
488
232
|
this._resetReplayPrefillRuntimeState();
|
|
@@ -490,120 +234,58 @@ export class PipelineGenerator {
|
|
|
490
234
|
}
|
|
491
235
|
|
|
492
236
|
async _replayPrefillDecodeLogits(currentIds, opts) {
|
|
493
|
-
|
|
494
|
-
// Without KV cache creation, this bound is not enforced elsewhere.
|
|
495
|
-
const kvConfig = this.#state.runtimeConfig?.inference?.session?.kvcache;
|
|
496
|
-
const replayMaxSeqLen = kvConfig?.maxSeqLen;
|
|
497
|
-
if (Number.isFinite(replayMaxSeqLen) && replayMaxSeqLen > 0 && currentIds.length > replayMaxSeqLen) {
|
|
498
|
-
throw new Error(
|
|
499
|
-
`[Pipeline] Replay-prefill sequence length ${currentIds.length} exceeds ` +
|
|
500
|
-
`runtime.inference.session.kvcache.maxSeqLen (${replayMaxSeqLen}). ` +
|
|
501
|
-
'Increase maxSeqLen in a tier profile or runtime config to allow longer sequences.'
|
|
502
|
-
);
|
|
503
|
-
}
|
|
504
|
-
this.#state.decodeStepCount++;
|
|
505
|
-
this._resetReplayPrefillRuntimeState();
|
|
506
|
-
const logits = await this._prefill(currentIds, opts);
|
|
507
|
-
return {
|
|
508
|
-
logits,
|
|
509
|
-
logitsBuffer: null,
|
|
510
|
-
logitsDtype: null,
|
|
511
|
-
rawVocabSize: this.#state.modelConfig.vocabSize,
|
|
512
|
-
vocabSize: this.#state.modelConfig.vocabSize,
|
|
513
|
-
};
|
|
237
|
+
return _replayPrefillDecodeLogitsImpl.apply(this, arguments);
|
|
514
238
|
}
|
|
515
239
|
|
|
516
240
|
_shouldUseFinitenessFallback(error, contextLabel) {
|
|
517
|
-
|
|
518
|
-
return false;
|
|
519
|
-
}
|
|
520
|
-
if (!hasFallbackExecutionPlan(this.#state)) {
|
|
521
|
-
throw createUnhandledFinitenessPolicyError(this.#state, contextLabel, error);
|
|
522
|
-
}
|
|
523
|
-
return true;
|
|
241
|
+
return _shouldUseFinitenessFallbackImpl.apply(this, arguments);
|
|
524
242
|
}
|
|
525
243
|
|
|
526
244
|
_recreateKVCacheForExecutionPlan(plan, reasonLabel) {
|
|
527
|
-
|
|
528
|
-
const runtimeInference = cloneRuntimeInferenceWithKVDtype(this.#state, kvDtype);
|
|
529
|
-
this.#state.kvCache?.destroy?.();
|
|
530
|
-
this.#state.kvCache = createKVCache(
|
|
531
|
-
this.#state.modelConfig,
|
|
532
|
-
this.#state.useGPU,
|
|
533
|
-
this.#state.debug,
|
|
534
|
-
runtimeInference
|
|
535
|
-
);
|
|
536
|
-
this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
|
|
537
|
-
this.#state.currentSeqLen = 0;
|
|
538
|
-
return kvDtype;
|
|
245
|
+
return _recreateKVCacheForExecutionPlanImpl.apply(this, arguments);
|
|
539
246
|
}
|
|
540
247
|
|
|
541
248
|
_openFinitenessFallbackWindow(opts, reasonLabel, tokenCount, rollbackSeqLen = undefined) {
|
|
542
|
-
|
|
543
|
-
? Math.max(1, Math.floor(tokenCount))
|
|
544
|
-
: 1;
|
|
545
|
-
if (this.#finitenessFallbackWindow) {
|
|
546
|
-
this.#finitenessFallbackWindow.remainingTokens = Math.max(
|
|
547
|
-
this.#finitenessFallbackWindow.remainingTokens,
|
|
548
|
-
normalizedCount
|
|
549
|
-
);
|
|
550
|
-
return;
|
|
551
|
-
}
|
|
552
|
-
const original = this._beginFinitenessFallback(opts, reasonLabel, rollbackSeqLen);
|
|
553
|
-
this.#finitenessFallbackWindow = {
|
|
554
|
-
original,
|
|
555
|
-
remainingTokens: normalizedCount,
|
|
556
|
-
};
|
|
249
|
+
return _openFinitenessFallbackWindowImpl.apply(this, arguments);
|
|
557
250
|
}
|
|
558
251
|
|
|
559
252
|
_closeFinitenessFallbackWindow(opts) {
|
|
560
|
-
|
|
561
|
-
return;
|
|
562
|
-
}
|
|
563
|
-
const original = this.#finitenessFallbackWindow.original;
|
|
564
|
-
this.#finitenessFallbackWindow = null;
|
|
565
|
-
this._endFinitenessFallback(opts, original);
|
|
253
|
+
return _closeFinitenessFallbackWindowImpl.apply(this, arguments);
|
|
566
254
|
}
|
|
567
255
|
|
|
568
256
|
_consumeFinitenessFallbackToken(opts) {
|
|
569
|
-
|
|
570
|
-
return;
|
|
571
|
-
}
|
|
572
|
-
this.#finitenessFallbackWindow.remainingTokens -= 1;
|
|
573
|
-
if (this.#finitenessFallbackWindow.remainingTokens <= 0) {
|
|
574
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
575
|
-
}
|
|
257
|
+
return _consumeFinitenessFallbackTokenImpl.apply(this, arguments);
|
|
576
258
|
}
|
|
577
259
|
|
|
578
260
|
_resolveStepOptions(options = {}) {
|
|
579
|
-
return resolveStepOptions(this
|
|
261
|
+
return resolveStepOptions(this._state, options);
|
|
580
262
|
}
|
|
581
263
|
|
|
582
264
|
_resetDecodeRuntimeState() {
|
|
583
|
-
this
|
|
584
|
-
this
|
|
585
|
-
this
|
|
586
|
-
this
|
|
587
|
-
this
|
|
588
|
-
this
|
|
589
|
-
this
|
|
590
|
-
this
|
|
591
|
-
this
|
|
592
|
-
this
|
|
593
|
-
this
|
|
594
|
-
this
|
|
595
|
-
this
|
|
596
|
-
this
|
|
597
|
-
this
|
|
598
|
-
this
|
|
599
|
-
this
|
|
600
|
-
this
|
|
601
|
-
this
|
|
602
|
-
this
|
|
603
|
-
this
|
|
604
|
-
this
|
|
605
|
-
this
|
|
606
|
-
this
|
|
265
|
+
this._state.stats.prefillProfileSteps = [];
|
|
266
|
+
this._state.stats.decodeMode = null;
|
|
267
|
+
this._state.stats.batchGuardReason = null;
|
|
268
|
+
this._state.stats.decodeProfileSteps = [];
|
|
269
|
+
this._state.stats.ttftMs = 0;
|
|
270
|
+
this._state.stats.decodeTimeMs = 0;
|
|
271
|
+
this._state.stats.prefillRecordMs = 0;
|
|
272
|
+
this._state.stats.prefillRecordOps = 0;
|
|
273
|
+
this._state.stats.prefillRecordPasses = 0;
|
|
274
|
+
this._state.stats.prefillRecordOpLabels = {};
|
|
275
|
+
this._state.stats.prefillSubmitWaitMs = 0;
|
|
276
|
+
this._state.stats.decodeRecordMs = 0;
|
|
277
|
+
this._state.stats.decodeRecordOps = 0;
|
|
278
|
+
this._state.stats.decodeRecordPasses = 0;
|
|
279
|
+
this._state.stats.decodeRecordOpLabels = {};
|
|
280
|
+
this._state.stats.decodeSubmitWaitMs = 0;
|
|
281
|
+
this._state.stats.decodeReadbackWaitMs = 0;
|
|
282
|
+
this._state.stats.decodeReadbackMapWaitMs = 0;
|
|
283
|
+
this._state.stats.decodeReadbackCleanupMs = 0;
|
|
284
|
+
this._state.stats.decodeReadbackCopyMs = 0;
|
|
285
|
+
this._state.decodeStepCount = 0;
|
|
286
|
+
this._state.disableRecordedLogits = false;
|
|
287
|
+
this._state.disableFusedDecode = false;
|
|
288
|
+
this._state.batchingStats = {
|
|
607
289
|
batchedForwardCalls: 0,
|
|
608
290
|
unbatchedForwardCalls: 0,
|
|
609
291
|
totalBatchedTimeMs: 0,
|
|
@@ -616,23 +298,23 @@ export class PipelineGenerator {
|
|
|
616
298
|
maxBatchTokenCap: null,
|
|
617
299
|
batchClampCount: 0,
|
|
618
300
|
};
|
|
619
|
-
resetActiveExecutionPlan(this
|
|
620
|
-
this
|
|
301
|
+
resetActiveExecutionPlan(this._state);
|
|
302
|
+
this._state.decodeRing?.reset();
|
|
621
303
|
}
|
|
622
304
|
|
|
623
305
|
_getDecodeHelpers(debugCheckBuffer) {
|
|
624
306
|
return {
|
|
625
307
|
buildLayerContext: (recorder, isDecodeMode, debugLayers, executionPlan) =>
|
|
626
|
-
buildLayerContext(this
|
|
627
|
-
getLogitsWeights: () => getLogitsWeights(this
|
|
628
|
-
getLogitsConfig: () => getLogitsConfig(this
|
|
308
|
+
buildLayerContext(this._state, recorder, isDecodeMode, debugLayers, debugCheckBuffer, executionPlan),
|
|
309
|
+
getLogitsWeights: () => getLogitsWeights(this._state),
|
|
310
|
+
getLogitsConfig: () => getLogitsConfig(this._state),
|
|
629
311
|
releaseSharedAttentionState,
|
|
630
312
|
debugCheckBuffer,
|
|
631
313
|
};
|
|
632
314
|
}
|
|
633
315
|
|
|
634
316
|
async _getFinalNormWeights() {
|
|
635
|
-
return getFinalNormWeights(this
|
|
317
|
+
return getFinalNormWeights(this._state);
|
|
636
318
|
}
|
|
637
319
|
|
|
638
320
|
_extractEmbeddingFromHidden(hiddenStates, numTokens, hiddenSize, embeddingMode, finalNormWeights, config) {
|
|
@@ -643,9 +325,9 @@ export class PipelineGenerator {
|
|
|
643
325
|
embeddingMode,
|
|
644
326
|
finalNormWeights,
|
|
645
327
|
config,
|
|
646
|
-
this
|
|
328
|
+
this._state.embeddingPostprocessor,
|
|
647
329
|
null,
|
|
648
|
-
this
|
|
330
|
+
this._state.weights.get('final_norm_bias') ?? null
|
|
649
331
|
);
|
|
650
332
|
}
|
|
651
333
|
|
|
@@ -656,13 +338,13 @@ export class PipelineGenerator {
|
|
|
656
338
|
hiddenSize,
|
|
657
339
|
finalNormWeights,
|
|
658
340
|
config,
|
|
659
|
-
this
|
|
341
|
+
this._state.weights.get('final_norm_bias') ?? null
|
|
660
342
|
);
|
|
661
343
|
}
|
|
662
344
|
|
|
663
345
|
_resolvePromptTokenIds(prompt, useChatTemplate, contextLabel) {
|
|
664
|
-
const processedPrompt = resolvePromptInput(this
|
|
665
|
-
const inputIds = this
|
|
346
|
+
const processedPrompt = resolvePromptInput(this._state, prompt, useChatTemplate, contextLabel);
|
|
347
|
+
const inputIds = this._state.tokenizer.encode(processedPrompt);
|
|
666
348
|
this._assertTokenIdsInRange(inputIds, `${contextLabel}.encode`);
|
|
667
349
|
return inputIds;
|
|
668
350
|
}
|
|
@@ -686,14 +368,14 @@ export class PipelineGenerator {
|
|
|
686
368
|
try {
|
|
687
369
|
opts.logitMaskFn(sampledLogits, {
|
|
688
370
|
generatedIds,
|
|
689
|
-
tokenizer: this
|
|
690
|
-
vocabSize: this
|
|
371
|
+
tokenizer: this._state.tokenizer ?? null,
|
|
372
|
+
vocabSize: this._state.modelConfig?.vocabSize ?? sampledLogits.length,
|
|
691
373
|
});
|
|
692
374
|
} catch (maskError) {
|
|
693
375
|
log.warn("Pipeline", `logitMaskFn threw; continuing without mask: ${maskError}`);
|
|
694
376
|
}
|
|
695
377
|
}
|
|
696
|
-
const padTokenId = this
|
|
378
|
+
const padTokenId = this._state.tokenizer?.getSpecialTokens?.()?.pad;
|
|
697
379
|
const tokenId = sample(sampledLogits, {
|
|
698
380
|
temperature: opts.temperature,
|
|
699
381
|
topP: opts.topP,
|
|
@@ -712,7 +394,7 @@ export class PipelineGenerator {
|
|
|
712
394
|
}
|
|
713
395
|
|
|
714
396
|
_resolvePrefillTokenChunkSize(inputIds) {
|
|
715
|
-
const chunkSize = resolveEffectivePrefillTokenChunkSize(this
|
|
397
|
+
const chunkSize = resolveEffectivePrefillTokenChunkSize(this._state);
|
|
716
398
|
if (chunkSize === undefined) {
|
|
717
399
|
throw new Error('inference.session.prefillTokenChunkSize is required; use null to disable token-chunked prefill.');
|
|
718
400
|
}
|
|
@@ -726,105 +408,26 @@ export class PipelineGenerator {
|
|
|
726
408
|
}
|
|
727
409
|
|
|
728
410
|
async _commitPrefillHiddenChunk(prefillResult) {
|
|
729
|
-
|
|
730
|
-
numTokens,
|
|
731
|
-
startPos,
|
|
732
|
-
currentRecorder,
|
|
733
|
-
recordProfile,
|
|
734
|
-
currentHiddenBuffer,
|
|
735
|
-
} = prefillResult;
|
|
736
|
-
|
|
737
|
-
try {
|
|
738
|
-
if (currentRecorder) {
|
|
739
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
740
|
-
await currentRecorder.submitAndWait();
|
|
741
|
-
await recordProfile(currentRecorder);
|
|
742
|
-
} else {
|
|
743
|
-
const device = getDevice();
|
|
744
|
-
if (device) {
|
|
745
|
-
await device.queue.onSubmittedWorkDone();
|
|
746
|
-
}
|
|
747
|
-
}
|
|
748
|
-
this.#state.currentSeqLen = startPos + numTokens;
|
|
749
|
-
} finally {
|
|
750
|
-
releaseBuffer(currentHiddenBuffer);
|
|
751
|
-
}
|
|
411
|
+
return _commitPrefillHiddenChunkImpl.apply(this, arguments);
|
|
752
412
|
}
|
|
753
413
|
|
|
754
414
|
async _prefillInputIdsToLogits(inputIds, opts) {
|
|
755
|
-
|
|
756
|
-
if (chunkSize === null) {
|
|
757
|
-
return this._prefill(inputIds, opts);
|
|
758
|
-
}
|
|
759
|
-
|
|
760
|
-
for (let offset = 0; offset < inputIds.length; offset += chunkSize) {
|
|
761
|
-
const end = Math.min(offset + chunkSize, inputIds.length);
|
|
762
|
-
const chunk = inputIds.slice(offset, end);
|
|
763
|
-
const isFinalChunk = end === inputIds.length;
|
|
764
|
-
if (isFinalChunk) {
|
|
765
|
-
return this._prefill(chunk, opts);
|
|
766
|
-
}
|
|
767
|
-
const prefillResult = await this._prefillToHidden(chunk, opts);
|
|
768
|
-
await this._commitPrefillHiddenChunk(prefillResult);
|
|
769
|
-
}
|
|
770
|
-
|
|
771
|
-
return this._prefill(inputIds, opts);
|
|
415
|
+
return _prefillInputIdsToLogitsImpl.apply(this, arguments);
|
|
772
416
|
}
|
|
773
417
|
|
|
774
418
|
async _prefillPromptToLogits(prompt, opts, contextLabel) {
|
|
775
|
-
|
|
776
|
-
const inputStart = performance.now();
|
|
777
|
-
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, contextLabel, opts.inputIds);
|
|
778
|
-
const inputMs = performance.now() - inputStart;
|
|
779
|
-
if (opts.debug) {
|
|
780
|
-
log.debug('Pipeline', `${contextLabel}: ${inputIds.length} tokens`);
|
|
781
|
-
}
|
|
782
|
-
|
|
783
|
-
let logits;
|
|
784
|
-
const prefillStart = performance.now();
|
|
785
|
-
const runPrefill = () => this._prefillInputIdsToLogits(inputIds, opts);
|
|
786
|
-
try {
|
|
787
|
-
logits = await runPrefill();
|
|
788
|
-
} catch (error) {
|
|
789
|
-
if (!this._shouldUseFinitenessFallback(error, contextLabel)) {
|
|
790
|
-
throw error;
|
|
791
|
-
}
|
|
792
|
-
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during ${contextLabel}. Retrying with F32 precision.`);
|
|
793
|
-
logits = await this._retryWithPersistentFinitenessFallback(
|
|
794
|
-
opts,
|
|
795
|
-
contextLabel,
|
|
796
|
-
opts.maxTokens ?? 1,
|
|
797
|
-
runPrefill,
|
|
798
|
-
prefillStartSeqLen
|
|
799
|
-
);
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
return {
|
|
803
|
-
inputIds,
|
|
804
|
-
logits,
|
|
805
|
-
phase: {
|
|
806
|
-
inputMs,
|
|
807
|
-
prefillMs: performance.now() - prefillStart,
|
|
808
|
-
tokens: inputIds.length,
|
|
809
|
-
prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
|
|
810
|
-
prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
|
|
811
|
-
prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
|
|
812
|
-
prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
|
|
813
|
-
prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
|
|
814
|
-
gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
|
|
815
|
-
},
|
|
816
|
-
};
|
|
419
|
+
return _prefillPromptToLogitsImpl.apply(this, arguments);
|
|
817
420
|
}
|
|
818
421
|
|
|
819
422
|
async _decodeStepToLogits(currentIds, opts) {
|
|
820
|
-
if (usesReplayPrefillDecode(this
|
|
423
|
+
if (usesReplayPrefillDecode(this._state)) {
|
|
821
424
|
return this._replayPrefillDecodeLogits(currentIds, opts);
|
|
822
425
|
}
|
|
823
|
-
const debugCheckBuffer = this
|
|
426
|
+
const debugCheckBuffer = this._state.debug
|
|
824
427
|
? (buffer, label, numTokens, expectedDim) =>
|
|
825
|
-
debugCheckBufferHelper(this
|
|
428
|
+
debugCheckBufferHelper(this._state, buffer, label, numTokens, expectedDim)
|
|
826
429
|
: undefined;
|
|
827
|
-
return decodeStepLogits(this
|
|
430
|
+
return decodeStepLogits(this._state, currentIds, opts, this._getDecodeHelpers(debugCheckBuffer));
|
|
828
431
|
}
|
|
829
432
|
|
|
830
433
|
async _decodeNextTokenViaLogits(currentIds, opts) {
|
|
@@ -836,7 +439,7 @@ export class PipelineGenerator {
|
|
|
836
439
|
if (!Array.isArray(stopSequences) || stopSequences.length === 0) {
|
|
837
440
|
return false;
|
|
838
441
|
}
|
|
839
|
-
const fullText = this
|
|
442
|
+
const fullText = this._state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
840
443
|
return stopSequences.some((sequence) => fullText.endsWith(sequence));
|
|
841
444
|
}
|
|
842
445
|
|
|
@@ -848,429 +451,32 @@ export class PipelineGenerator {
|
|
|
848
451
|
}
|
|
849
452
|
|
|
850
453
|
_recordStopReason(reason, tokenId = null) {
|
|
851
|
-
this
|
|
852
|
-
this
|
|
454
|
+
this._state.stats.stopReason = reason;
|
|
455
|
+
this._state.stats.stopTokenId = Number.isInteger(tokenId) ? tokenId : null;
|
|
853
456
|
}
|
|
854
457
|
|
|
855
458
|
async *_generateTokensInternal(prompt, options = {}, mode = 'text') {
|
|
856
|
-
|
|
857
|
-
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
858
|
-
|
|
859
|
-
validateCallTimeOptions(options);
|
|
860
|
-
|
|
861
|
-
this.#state.isGenerating = true;
|
|
862
|
-
this._resetDecodeRuntimeState();
|
|
863
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
864
|
-
this.#state.stats.gpuTimeDecodeMs = undefined;
|
|
865
|
-
this.#state.stats.decodeRecordMs = 0;
|
|
866
|
-
this.#state.stats.decodeRecordOps = 0;
|
|
867
|
-
this.#state.stats.decodeRecordPasses = 0;
|
|
868
|
-
this.#state.stats.decodeRecordOpLabels = {};
|
|
869
|
-
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
870
|
-
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
871
|
-
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
872
|
-
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
873
|
-
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
874
|
-
this.#state.stats.prefillRecordMs = 0;
|
|
875
|
-
this.#state.stats.prefillRecordOps = 0;
|
|
876
|
-
this.#state.stats.prefillRecordPasses = 0;
|
|
877
|
-
this.#state.stats.prefillRecordOpLabels = {};
|
|
878
|
-
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
879
|
-
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
880
|
-
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
881
|
-
this.#state.stats.singleTokenReadbackMapWaitMs = 0;
|
|
882
|
-
this.#state.stats.singleTokenReadbackCleanupMs = 0;
|
|
883
|
-
this.#state.stats.singleTokenReadbackCopyMs = 0;
|
|
884
|
-
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
885
|
-
this.#state.stats.pleHotVocabularyHits = 0;
|
|
886
|
-
this.#state.stats.pleHotVocabularyMisses = 0;
|
|
887
|
-
this.#state.stats.plePreparedTokenCacheHits = 0;
|
|
888
|
-
this.#state.stats.plePreparedTokenCacheMisses = 0;
|
|
889
|
-
this.#state.stats.plePreparedTokenCacheEntries = 0;
|
|
890
|
-
this.#state.stats.plePreparedTokenCacheBytes = 0;
|
|
891
|
-
this.#state.stats.decodeMode = null;
|
|
892
|
-
this.#state.stats.batchGuardReason = null;
|
|
893
|
-
this.#state.stats.stopReason = null;
|
|
894
|
-
this.#state.stats.stopTokenId = null;
|
|
895
|
-
this.#state.stats.ttftMs = 0;
|
|
896
|
-
const startTime = performance.now();
|
|
897
|
-
|
|
898
|
-
const opts = resolveGenerateOptions(this.#state, options);
|
|
899
|
-
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
900
|
-
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
901
|
-
// Validate and normalize sampling parameters through single source of truth
|
|
902
|
-
const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
|
|
903
|
-
opts.temperature = samplingConfig.temperature;
|
|
904
|
-
opts.topP = samplingConfig.topP;
|
|
905
|
-
opts.topK = samplingConfig.topK;
|
|
906
|
-
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
907
|
-
opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this.#state, samplingConfig);
|
|
908
|
-
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
909
|
-
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
910
|
-
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
911
|
-
if (diagnosticsEnabled || tsirFixtureCfg) {
|
|
912
|
-
const captureConfig = {
|
|
913
|
-
...createDefaultCaptureConfig(),
|
|
914
|
-
enabled: true,
|
|
915
|
-
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
916
|
-
...(options?.diagnostics?.captureConfig ?? {}),
|
|
917
|
-
};
|
|
918
|
-
validateCaptureConfig(captureConfig);
|
|
919
|
-
this.#state.operatorDiagnostics = {
|
|
920
|
-
enabled: diagnosticsEnabled === true,
|
|
921
|
-
captureConfig,
|
|
922
|
-
emitter: diagnosticsEnabled ? new OperatorEventEmitter({
|
|
923
|
-
modelHash: this.#state.manifest?.modelId ?? null,
|
|
924
|
-
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
925
|
-
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
926
|
-
}) : null,
|
|
927
|
-
tsirFixture: tsirFixtureCfg ? {
|
|
928
|
-
dir: tsirFixtureCfg.dir,
|
|
929
|
-
layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
|
|
930
|
-
? tsirFixtureCfg.layerFilter
|
|
931
|
-
: null,
|
|
932
|
-
records: [],
|
|
933
|
-
} : null,
|
|
934
|
-
};
|
|
935
|
-
}
|
|
936
|
-
const activePlan = opts.executionPlan ?? resolveActiveExecutionPlan(this.#state);
|
|
937
|
-
this.#state.stats.executionPlan = {
|
|
938
|
-
primary: summarizeExecutionPlan(this.#state.executionPlanState?.primaryPlan ?? null),
|
|
939
|
-
fallback: summarizeExecutionPlan(this.#state.executionPlanState?.fallbackPlan ?? null),
|
|
940
|
-
activePlanIdAtStart: activePlan?.id ?? null,
|
|
941
|
-
finalActivePlanId: activePlan?.id ?? null,
|
|
942
|
-
transitions: [],
|
|
943
|
-
};
|
|
944
|
-
this.#state.stats.kernelPathId = activePlan?.kernelPathId ?? this.#state.resolvedKernelPath?.id ?? null;
|
|
945
|
-
this.#state.stats.kernelPathSource = activePlan?.kernelPathSource ?? this.#state.kernelPathSource ?? 'none';
|
|
946
|
-
|
|
947
|
-
if (opts.debug) {
|
|
948
|
-
log.debug('Pipeline', `ChatTemplate: options=${options.useChatTemplate}, final=${opts.useChatTemplate}`);
|
|
949
|
-
}
|
|
950
|
-
|
|
951
|
-
const emitToken = async function* (generator, tokenId, textDecoder) {
|
|
952
|
-
if (mode === 'token') {
|
|
953
|
-
yield tokenId;
|
|
954
|
-
if (options.onToken) options.onToken(tokenId, '');
|
|
955
|
-
return;
|
|
956
|
-
}
|
|
957
|
-
const tokenText = textDecoder(tokenId);
|
|
958
|
-
yield tokenText;
|
|
959
|
-
if (options.onToken) options.onToken(tokenId, tokenText);
|
|
960
|
-
};
|
|
961
|
-
|
|
962
|
-
try {
|
|
963
|
-
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
964
|
-
const prefillStart = performance.now();
|
|
965
|
-
const { inputIds, logits: initialPrefillLogits } = await this._prefillPromptToLogits(prompt, opts, 'generate');
|
|
966
|
-
let prefillLogits = initialPrefillLogits;
|
|
967
|
-
this.#state.stats.prefillTimeMs = performance.now() - prefillStart;
|
|
968
|
-
this._assertTokenIdsInRange(inputIds, 'generate.prefillTokens');
|
|
969
|
-
const generatedIds = [...inputIds];
|
|
970
|
-
this.#state.stats.prefillTokens = inputIds.length;
|
|
971
|
-
|
|
972
|
-
if (opts.debug) {
|
|
973
|
-
log.debug('Pipeline', `Input: ${inputIds.length} tokens`);
|
|
974
|
-
}
|
|
975
|
-
|
|
976
|
-
const intentBundleConfig = this.#state.runtimeConfig.shared.intentBundle;
|
|
977
|
-
const intentBundle = intentBundleConfig?.bundle;
|
|
978
|
-
const expectedTopK = intentBundle?.payload?.expectedTopK
|
|
979
|
-
?? intentBundle?.payload?.expected_top_k;
|
|
980
|
-
const maxDriftThreshold = intentBundle?.constraints?.maxDriftThreshold
|
|
981
|
-
?? intentBundle?.constraints?.max_drift_threshold;
|
|
982
|
-
|
|
983
|
-
if (intentBundleConfig?.enabled && Array.isArray(expectedTopK) && expectedTopK.length > 0) {
|
|
984
|
-
const { enforceLogitDrift } = await getExperimentalIntentBundleModule();
|
|
985
|
-
const actualTopK = getTopK(
|
|
986
|
-
prefillLogits,
|
|
987
|
-
expectedTopK.length,
|
|
988
|
-
(tokens) => resolveTokenText(this.#state.tokenizer, tokens),
|
|
989
|
-
).map((token) => token.token);
|
|
990
|
-
const driftResult = enforceLogitDrift(expectedTopK, actualTopK, maxDriftThreshold);
|
|
991
|
-
if (!driftResult.ok) {
|
|
992
|
-
throw new Error(`Intent bundle drift check failed: ${driftResult.reason}`);
|
|
993
|
-
}
|
|
994
|
-
}
|
|
995
|
-
|
|
996
|
-
if (opts.debug) {
|
|
997
|
-
const topAfterPenalty = getTopK(
|
|
998
|
-
Float32Array.from(prefillLogits),
|
|
999
|
-
5,
|
|
1000
|
-
(tokens) => resolveTokenText(this.#state.tokenizer, tokens)
|
|
1001
|
-
);
|
|
1002
|
-
log.debug('Pipeline', `After rep penalty top-5: ${topAfterPenalty.map(t => `"${t.text}"(${(t.prob * 100).toFixed(1)}%)`).join(', ')}`);
|
|
1003
|
-
}
|
|
1004
|
-
|
|
1005
|
-
let firstToken;
|
|
1006
|
-
try {
|
|
1007
|
-
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
1008
|
-
} catch (error) {
|
|
1009
|
-
if (!this._shouldUseFinitenessFallback(error, 'prefill-sample')) {
|
|
1010
|
-
throw error;
|
|
1011
|
-
}
|
|
1012
|
-
log.warn('Pipeline', 'FinitenessGuard caught non-finite prefill logits at sampling. Retrying with F32 precision.');
|
|
1013
|
-
prefillLogits = await this._retryWithPersistentFinitenessFallback(
|
|
1014
|
-
opts,
|
|
1015
|
-
'prefill-sample',
|
|
1016
|
-
opts.maxTokens,
|
|
1017
|
-
() => this._prefill(inputIds, opts),
|
|
1018
|
-
prefillStartSeqLen
|
|
1019
|
-
);
|
|
1020
|
-
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
1021
|
-
}
|
|
1022
|
-
|
|
1023
|
-
if (opts.debug) {
|
|
1024
|
-
const firstTokenText = resolveTokenText(this.#state.tokenizer, [firstToken], `[${firstToken}]`, (tokens) => this.#state.tokenizer?.decode?.(tokens, true, false));
|
|
1025
|
-
log.debug('Pipeline', `First token sampled: id=${firstToken} text="${firstTokenText}"`);
|
|
1026
|
-
}
|
|
1027
|
-
|
|
1028
|
-
const stopTokenIds = this.#state.modelConfig.stopTokenIds;
|
|
1029
|
-
const eosToken = this.#state.tokenizer.getSpecialTokens?.()?.eos;
|
|
1030
|
-
const stopSequenceStart = inputIds.length;
|
|
1031
|
-
generatedIds.push(firstToken);
|
|
1032
|
-
this.#state.stats.ttftMs = performance.now() - startTime;
|
|
1033
|
-
|
|
1034
|
-
const decodeToken = (tokenId) => resolveTokenText(
|
|
1035
|
-
this.#state.tokenizer,
|
|
1036
|
-
[tokenId],
|
|
1037
|
-
`[${tokenId}]`,
|
|
1038
|
-
(tokens) => this.#state.tokenizer?.decode?.(tokens, true, false),
|
|
1039
|
-
(tokens) => this.#state.tokenizer?.decode?.(tokens, false, false)
|
|
1040
|
-
);
|
|
1041
|
-
const decodeRuntime = {
|
|
1042
|
-
stopTokenIds,
|
|
1043
|
-
eosToken,
|
|
1044
|
-
stopSequenceStart,
|
|
1045
|
-
decodeToken,
|
|
1046
|
-
logBatchPath: opts.debug,
|
|
1047
|
-
emitMode: mode,
|
|
1048
|
-
};
|
|
1049
|
-
|
|
1050
|
-
yield* emitToken(this, firstToken, decodeToken);
|
|
1051
|
-
|
|
1052
|
-
if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
|
|
1053
|
-
this._recordStopReason('stop-token-or-sequence', firstToken);
|
|
1054
|
-
this.#state.stats.decodeTimeMs = 0;
|
|
1055
|
-
this.#state.stats.tokensGenerated = 1;
|
|
1056
|
-
this.#state.stats.decodeTokens = 1;
|
|
1057
|
-
} else {
|
|
1058
|
-
yield* this._runDecodeLoop(generatedIds, opts, options, decodeRuntime);
|
|
1059
|
-
}
|
|
1060
|
-
const tokensGenerated = this.#state.stats.decodeTokens ?? 1;
|
|
1061
|
-
this.#state.stats.totalTimeMs = performance.now() - startTime;
|
|
1062
|
-
|
|
1063
|
-
if (opts.debug) {
|
|
1064
|
-
log.debug('Pipeline', `Generated ${tokensGenerated} tokens in ${this.#state.stats.totalTimeMs.toFixed(0)}ms`);
|
|
1065
|
-
}
|
|
1066
|
-
|
|
1067
|
-
const ttft = this.#state.stats.ttftMs ?? this.#state.stats.prefillTimeMs;
|
|
1068
|
-
const decodeTokens = Math.max(0, tokensGenerated - 1);
|
|
1069
|
-
const decodeSpeed = decodeTokens > 0 ? (decodeTokens / this.#state.stats.decodeTimeMs * 1000) : 0;
|
|
1070
|
-
const loadMs = this.#state.stats.modelLoadMs;
|
|
1071
|
-
const loadLabel = Number.isFinite(loadMs) ? `Load: ${loadMs.toFixed(0)}ms | ` : '';
|
|
1072
|
-
if (opts.benchmark) {
|
|
1073
|
-
log.info('Benchmark', `${loadLabel}TTFT: ${ttft.toFixed(0)}ms | Prefill: ${this.#state.stats.prefillTimeMs.toFixed(0)}ms | Decode: ${this.#state.stats.decodeTimeMs.toFixed(0)}ms (${decodeTokens} tokens @ ${decodeSpeed.toFixed(1)} tok/s)`);
|
|
1074
|
-
} else {
|
|
1075
|
-
log.info('Perf', `${loadLabel}TTFT: ${ttft.toFixed(0)}ms | Prefill: ${this.#state.stats.prefillTimeMs.toFixed(0)}ms | Decode: ${this.#state.stats.decodeTimeMs.toFixed(0)}ms (${decodeTokens} tokens @ ${decodeSpeed.toFixed(1)} tok/s)`);
|
|
1076
|
-
}
|
|
1077
|
-
trace.perf('Decode summary', {
|
|
1078
|
-
ttftMs: ttft,
|
|
1079
|
-
prefillMs: this.#state.stats.prefillTimeMs,
|
|
1080
|
-
decodeMs: this.#state.stats.decodeTimeMs,
|
|
1081
|
-
decodeTokens,
|
|
1082
|
-
decodeSpeed,
|
|
1083
|
-
totalMs: this.#state.stats.totalTimeMs,
|
|
1084
|
-
});
|
|
1085
|
-
} finally {
|
|
1086
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
1087
|
-
resetActiveExecutionPlan(this.#state);
|
|
1088
|
-
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
1089
|
-
? {
|
|
1090
|
-
enabled: true,
|
|
1091
|
-
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
1092
|
-
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
1093
|
-
}
|
|
1094
|
-
: null;
|
|
1095
|
-
this.#state.operatorDiagnostics = null;
|
|
1096
|
-
this.#state.isGenerating = false;
|
|
1097
|
-
}
|
|
459
|
+
return yield* _generateTokensInternalImpl.apply(this, arguments);
|
|
1098
460
|
}
|
|
1099
461
|
|
|
1100
462
|
_beginFinitenessFallback(opts, reasonLabel, rollbackSeqLen = undefined) {
|
|
1101
|
-
|
|
1102
|
-
const currentKvDtype = resolveCurrentKVCacheDtype(
|
|
1103
|
-
this.#state,
|
|
1104
|
-
originalPlan,
|
|
1105
|
-
`${reasonLabel}: current plan`
|
|
1106
|
-
);
|
|
1107
|
-
const original = {
|
|
1108
|
-
activePlanId: this.#state.executionPlanState?.activePlanId ?? 'primary',
|
|
1109
|
-
seed: opts.seed,
|
|
1110
|
-
restoreKVCachePlan: null,
|
|
1111
|
-
};
|
|
1112
|
-
|
|
1113
|
-
const fallbackPlan = activateFallbackExecutionPlan(this.#state);
|
|
1114
|
-
if (!fallbackPlan) {
|
|
1115
|
-
throw new Error(
|
|
1116
|
-
'[Pipeline] Explicit alternate-plan finiteness recovery is unavailable for this model/runtime configuration.'
|
|
1117
|
-
);
|
|
1118
|
-
}
|
|
1119
|
-
log.warn(
|
|
1120
|
-
'Pipeline',
|
|
1121
|
-
`FinitenessGuard fallback (${reasonLabel}): ` +
|
|
1122
|
-
`${originalPlan.kernelPathId ?? 'none'} -> ${fallbackPlan.kernelPathId ?? 'none'}`
|
|
1123
|
-
);
|
|
1124
|
-
const fallbackKvDtype = resolveTargetPlanKVDtype(
|
|
1125
|
-
fallbackPlan,
|
|
1126
|
-
`${reasonLabel}: fallback plan`
|
|
1127
|
-
);
|
|
1128
|
-
|
|
1129
|
-
if (Number.isInteger(rollbackSeqLen) && rollbackSeqLen < 0) {
|
|
1130
|
-
setActiveExecutionPlan(this.#state, original.activePlanId);
|
|
1131
|
-
throw new Error(
|
|
1132
|
-
`[Pipeline] ${reasonLabel}: rollbackSeqLen must be a non-negative integer when provided.`
|
|
1133
|
-
);
|
|
1134
|
-
}
|
|
1135
|
-
|
|
1136
|
-
if (fallbackKvDtype !== currentKvDtype) {
|
|
1137
|
-
if (rollbackSeqLen !== 0) {
|
|
1138
|
-
setActiveExecutionPlan(this.#state, original.activePlanId);
|
|
1139
|
-
throw new Error(
|
|
1140
|
-
`[Pipeline] ${reasonLabel}: finiteness fallback requires rebuilding the KV cache ` +
|
|
1141
|
-
`${currentKvDtype} -> ${fallbackKvDtype}, which is only supported from a fresh prefill (rollbackSeqLen=0).`
|
|
1142
|
-
);
|
|
1143
|
-
}
|
|
1144
|
-
try {
|
|
1145
|
-
this._recreateKVCacheForExecutionPlan(fallbackPlan, reasonLabel);
|
|
1146
|
-
original.restoreKVCachePlan = originalPlan;
|
|
1147
|
-
} catch (error) {
|
|
1148
|
-
setActiveExecutionPlan(this.#state, original.activePlanId);
|
|
1149
|
-
try {
|
|
1150
|
-
this._recreateKVCacheForExecutionPlan(originalPlan, `${reasonLabel}: restore primary`);
|
|
1151
|
-
} catch (restoreError) {
|
|
1152
|
-
log.warn(
|
|
1153
|
-
'Pipeline',
|
|
1154
|
-
`Failed to restore primary KV cache after fallback activation error: ${restoreError}`
|
|
1155
|
-
);
|
|
1156
|
-
}
|
|
1157
|
-
throw error;
|
|
1158
|
-
}
|
|
1159
|
-
} else if (Number.isInteger(rollbackSeqLen)) {
|
|
1160
|
-
this.#state.kvCache?.truncate(rollbackSeqLen);
|
|
1161
|
-
this.#state.currentSeqLen = rollbackSeqLen;
|
|
1162
|
-
if (rollbackSeqLen === 0) {
|
|
1163
|
-
this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
|
|
1164
|
-
}
|
|
1165
|
-
} else {
|
|
1166
|
-
this.#state.kvCache?.truncate(this.#state.currentSeqLen);
|
|
1167
|
-
}
|
|
1168
|
-
|
|
1169
|
-
this.#state.decodeBuffers?.ensureBuffers({
|
|
1170
|
-
hiddenSize: this.#state.modelConfig.hiddenSize,
|
|
1171
|
-
intermediateSize: this.#state.modelConfig.maxIntermediateSize,
|
|
1172
|
-
activationDtype: fallbackPlan.activationDtype,
|
|
1173
|
-
enablePingPong: true,
|
|
1174
|
-
});
|
|
1175
|
-
|
|
1176
|
-
if (opts.seed == null) {
|
|
1177
|
-
const fallbackSeedBase = (this.#state.decodeStepCount + this.#state.currentSeqLen + 1) >>> 0;
|
|
1178
|
-
opts.seed = (fallbackSeedBase * 2654435761) >>> 0;
|
|
1179
|
-
}
|
|
1180
|
-
opts.executionPlan = rebaseExecutionSessionPlan(this.#state, opts.executionPlan);
|
|
1181
|
-
if (this.#state.stats.executionPlan) {
|
|
1182
|
-
this.#state.stats.executionPlan.finalActivePlanId = fallbackPlan.id;
|
|
1183
|
-
this.#state.stats.executionPlan.transitions.push({
|
|
1184
|
-
kind: 'activate-finiteness-fallback',
|
|
1185
|
-
reason: reasonLabel ?? null,
|
|
1186
|
-
decodeStep: this.#state.decodeStepCount,
|
|
1187
|
-
seqLen: this.#state.currentSeqLen,
|
|
1188
|
-
fromPlanId: originalPlan.id,
|
|
1189
|
-
toPlanId: fallbackPlan.id,
|
|
1190
|
-
fromKernelPathId: originalPlan.kernelPathId ?? null,
|
|
1191
|
-
toKernelPathId: fallbackPlan.kernelPathId ?? null,
|
|
1192
|
-
});
|
|
1193
|
-
}
|
|
1194
|
-
this.#state.stats.kernelPathId = fallbackPlan.kernelPathId ?? null;
|
|
1195
|
-
this.#state.stats.kernelPathSource = fallbackPlan.kernelPathSource ?? 'none';
|
|
1196
|
-
|
|
1197
|
-
return original;
|
|
463
|
+
return _beginFinitenessFallbackImpl.apply(this, arguments);
|
|
1198
464
|
}
|
|
1199
465
|
|
|
1200
466
|
_endFinitenessFallback(opts, original) {
|
|
1201
|
-
|
|
1202
|
-
setActiveExecutionPlan(this.#state, original.activePlanId);
|
|
1203
|
-
opts.executionPlan = rebaseExecutionSessionPlan(this.#state, opts.executionPlan);
|
|
1204
|
-
const restoredPlan = resolveActiveExecutionPlan(this.#state);
|
|
1205
|
-
if (original.restoreKVCachePlan) {
|
|
1206
|
-
this._recreateKVCacheForExecutionPlan(restoredPlan, 'restore-primary-plan');
|
|
1207
|
-
}
|
|
1208
|
-
if (this.#state.stats.executionPlan) {
|
|
1209
|
-
this.#state.stats.executionPlan.finalActivePlanId = restoredPlan.id;
|
|
1210
|
-
this.#state.stats.executionPlan.transitions.push({
|
|
1211
|
-
kind: 'restore-primary-plan',
|
|
1212
|
-
reason: null,
|
|
1213
|
-
decodeStep: this.#state.decodeStepCount,
|
|
1214
|
-
seqLen: this.#state.currentSeqLen,
|
|
1215
|
-
fromPlanId: this.#state.executionPlanState?.fallbackPlan?.id ?? null,
|
|
1216
|
-
toPlanId: restoredPlan.id,
|
|
1217
|
-
fromKernelPathId: this.#state.executionPlanState?.fallbackPlan?.kernelPathId ?? null,
|
|
1218
|
-
toKernelPathId: restoredPlan.kernelPathId ?? null,
|
|
1219
|
-
});
|
|
1220
|
-
}
|
|
1221
|
-
this.#state.stats.kernelPathId = restoredPlan.kernelPathId ?? this.#state.resolvedKernelPath?.id ?? null;
|
|
1222
|
-
this.#state.stats.kernelPathSource = restoredPlan.kernelPathSource ?? this.#state.kernelPathSource ?? 'none';
|
|
1223
|
-
const nextActivationDtype = this._getEffectiveActivationDtype();
|
|
1224
|
-
this.#state.decodeBuffers?.ensureBuffers({
|
|
1225
|
-
hiddenSize: this.#state.modelConfig.hiddenSize,
|
|
1226
|
-
intermediateSize: this.#state.modelConfig.maxIntermediateSize,
|
|
1227
|
-
activationDtype: nextActivationDtype,
|
|
1228
|
-
enablePingPong: true,
|
|
1229
|
-
});
|
|
467
|
+
return _endFinitenessFallbackImpl.apply(this, arguments);
|
|
1230
468
|
}
|
|
1231
469
|
|
|
1232
470
|
async _retryWithFinitenessFallback(opts, reasonLabel, retryFn, rollbackSeqLen = undefined) {
|
|
1233
|
-
|
|
1234
|
-
return retryFn();
|
|
1235
|
-
}
|
|
1236
|
-
const original = this._beginFinitenessFallback(opts, reasonLabel, rollbackSeqLen);
|
|
1237
|
-
try {
|
|
1238
|
-
return await retryFn();
|
|
1239
|
-
} finally {
|
|
1240
|
-
this._endFinitenessFallback(opts, original);
|
|
1241
|
-
}
|
|
471
|
+
return _retryWithFinitenessFallbackImpl.apply(this, arguments);
|
|
1242
472
|
}
|
|
1243
473
|
|
|
1244
474
|
async _retryWithPersistentFinitenessFallback(opts, reasonLabel, tokenBudget, retryFn, rollbackSeqLen = undefined) {
|
|
1245
|
-
|
|
1246
|
-
return retryFn();
|
|
1247
|
-
}
|
|
1248
|
-
this._openFinitenessFallbackWindow(opts, reasonLabel, tokenBudget, rollbackSeqLen);
|
|
1249
|
-
try {
|
|
1250
|
-
return await retryFn();
|
|
1251
|
-
} catch (error) {
|
|
1252
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
1253
|
-
throw error;
|
|
1254
|
-
}
|
|
475
|
+
return _retryWithPersistentFinitenessFallbackImpl.apply(this, arguments);
|
|
1255
476
|
}
|
|
1256
477
|
|
|
1257
478
|
async _retryDecodeStepWithFinitenessWindow(generatedIds, opts, reasonLabel) {
|
|
1258
|
-
|
|
1259
|
-
if (windowTokens <= 1) {
|
|
1260
|
-
return this._retryWithFinitenessFallback(
|
|
1261
|
-
opts,
|
|
1262
|
-
reasonLabel,
|
|
1263
|
-
() => this._decodeStep(generatedIds, opts)
|
|
1264
|
-
);
|
|
1265
|
-
}
|
|
1266
|
-
|
|
1267
|
-
this._openFinitenessFallbackWindow(opts, reasonLabel, windowTokens);
|
|
1268
|
-
try {
|
|
1269
|
-
return await this._decodeStep(generatedIds, opts);
|
|
1270
|
-
} catch (error) {
|
|
1271
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
1272
|
-
throw error;
|
|
1273
|
-
}
|
|
479
|
+
return _retryDecodeStepWithFinitenessWindowImpl.apply(this, arguments);
|
|
1274
480
|
}
|
|
1275
481
|
|
|
1276
482
|
// ==========================================================================
|
|
@@ -1287,591 +493,36 @@ export class PipelineGenerator {
|
|
|
1287
493
|
* Only valid when no decode is in progress.
|
|
1288
494
|
*/
|
|
1289
495
|
resetToSeqLen(seqLen) {
|
|
1290
|
-
if (this
|
|
496
|
+
if (this._state.isGenerating) {
|
|
1291
497
|
throw new Error('InferencePipeline.resetToSeqLen: cannot reset while generation is in progress');
|
|
1292
498
|
}
|
|
1293
499
|
const target = Math.max(0, Math.floor(Number(seqLen) || 0));
|
|
1294
500
|
if (!Number.isFinite(target)) {
|
|
1295
501
|
throw new Error('InferencePipeline.resetToSeqLen: seqLen must be a finite non-negative integer');
|
|
1296
502
|
}
|
|
1297
|
-
if (target > this
|
|
503
|
+
if (target > this._state.currentSeqLen) {
|
|
1298
504
|
throw new Error(
|
|
1299
|
-
`InferencePipeline.resetToSeqLen: target ${target} exceeds currentSeqLen ${this
|
|
505
|
+
`InferencePipeline.resetToSeqLen: target ${target} exceeds currentSeqLen ${this._state.currentSeqLen}`
|
|
1300
506
|
);
|
|
1301
507
|
}
|
|
1302
|
-
this
|
|
1303
|
-
this
|
|
508
|
+
this._state.kvCache?.truncate?.(target);
|
|
509
|
+
this._state.currentSeqLen = target;
|
|
1304
510
|
}
|
|
1305
511
|
|
|
1306
512
|
async _createDiffusionGemmaSelfConditioningEmbeddings(canvasIds, selfConditioningLogits, opts) {
|
|
1307
|
-
|
|
1308
|
-
const canvasLength = canvasIds.length;
|
|
1309
|
-
const hiddenSize = config.hiddenSize;
|
|
1310
|
-
const vocabSize = config.vocabSize;
|
|
1311
|
-
const elementCount = canvasLength * hiddenSize;
|
|
1312
|
-
const softEmbeddingState = normalizeSelfConditioningSoftEmbeddingState(
|
|
1313
|
-
selfConditioningLogits,
|
|
1314
|
-
canvasLength,
|
|
1315
|
-
hiddenSize
|
|
1316
|
-
);
|
|
1317
|
-
const logitsState = softEmbeddingState
|
|
1318
|
-
? null
|
|
1319
|
-
: normalizeSelfConditioningLogitsState(selfConditioningLogits, canvasLength, vocabSize);
|
|
1320
|
-
const logits = (softEmbeddingState || logitsState)
|
|
1321
|
-
? null
|
|
1322
|
-
: normalizeSelfConditioningLogits(selfConditioningLogits, canvasLength, vocabSize);
|
|
1323
|
-
const weights = this.#state.weights.get('diffusion_gemma_self_conditioning');
|
|
1324
|
-
if (!weights || typeof weights !== 'object') {
|
|
1325
|
-
throw new Error(
|
|
1326
|
-
'DiffusionGemma self-conditioning weights were not loaded. ' +
|
|
1327
|
-
'Expected model.decoder.self_conditioning tensors in the manifest.'
|
|
1328
|
-
);
|
|
1329
|
-
}
|
|
1330
|
-
|
|
1331
|
-
const embedBufferRaw = this.#state.weights.get('embed');
|
|
1332
|
-
if (!embedBufferRaw) {
|
|
1333
|
-
throw new Error('DiffusionGemma self-conditioning requires loaded embed_tokens weights.');
|
|
1334
|
-
}
|
|
1335
|
-
const borrowed = {
|
|
1336
|
-
preNorm: null,
|
|
1337
|
-
gateProj: null,
|
|
1338
|
-
upProj: null,
|
|
1339
|
-
downProj: null,
|
|
1340
|
-
softEmbedding: null,
|
|
1341
|
-
};
|
|
1342
|
-
let softLogitsTensor = null;
|
|
1343
|
-
let softLogitsOwned = true;
|
|
1344
|
-
let softEmbeddingStateTensor = null;
|
|
1345
|
-
let softEmbeddingStateOwned = false;
|
|
1346
|
-
let softmaxTensor = null;
|
|
1347
|
-
let softEmbeddings = null;
|
|
1348
|
-
let scaledSoftEmbeddings = null;
|
|
1349
|
-
let baseEmbeddings = null;
|
|
1350
|
-
let normedSoft = null;
|
|
1351
|
-
let gate = null;
|
|
1352
|
-
let up = null;
|
|
1353
|
-
let activated = null;
|
|
1354
|
-
let down = null;
|
|
1355
|
-
let combined = null;
|
|
1356
|
-
let output = null;
|
|
1357
|
-
|
|
1358
|
-
const releaseTensorOnce = (() => {
|
|
1359
|
-
const released = new Set();
|
|
1360
|
-
return (tensor) => {
|
|
1361
|
-
const buffer = tensor?.buffer ?? null;
|
|
1362
|
-
if (!buffer || released.has(buffer)) return;
|
|
1363
|
-
released.add(buffer);
|
|
1364
|
-
releaseBuffer(buffer);
|
|
1365
|
-
};
|
|
1366
|
-
})();
|
|
1367
|
-
|
|
1368
|
-
try {
|
|
1369
|
-
borrowed.preNorm = borrowNormWeight(weights.preNorm, 'diffusion_gemma_self_conditioning.pre_norm');
|
|
1370
|
-
borrowed.gateProj = borrowLinearWeight(weights.gateProj, 'diffusion_gemma_self_conditioning.gate_proj');
|
|
1371
|
-
borrowed.upProj = borrowLinearWeight(weights.upProj, 'diffusion_gemma_self_conditioning.up_proj');
|
|
1372
|
-
borrowed.downProj = borrowLinearWeight(weights.downProj, 'diffusion_gemma_self_conditioning.down_proj');
|
|
1373
|
-
if (isSplitWeightBuffer(embedBufferRaw) && this.#state.embeddingTranspose === true) {
|
|
1374
|
-
throw new Error(
|
|
1375
|
-
'DiffusionGemma self-conditioning split embeddings require row-major embedding storage.'
|
|
1376
|
-
);
|
|
1377
|
-
}
|
|
1378
|
-
borrowed.softEmbedding = isSplitWeightBuffer(embedBufferRaw)
|
|
1379
|
-
? null
|
|
1380
|
-
: borrowLinearWeight(embedBufferRaw, 'diffusion_gemma_self_conditioning.embed_tokens');
|
|
1381
|
-
|
|
1382
|
-
const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
|
|
1383
|
-
const embedDtype = isCpuWeightBuffer(embedBufferRaw)
|
|
1384
|
-
? embedBufferRaw.dtype
|
|
1385
|
-
: getWeightDtype(embedBufferRaw);
|
|
1386
|
-
const embedMetadata = getWeightMetadata(embedBufferRaw);
|
|
1387
|
-
baseEmbeddings = await embed(canvasIds, embedBuffer, {
|
|
1388
|
-
hiddenSize,
|
|
1389
|
-
vocabSize,
|
|
1390
|
-
scaleEmbeddings: config.scaleEmbeddings,
|
|
1391
|
-
embeddingScale: config.embeddingScale,
|
|
1392
|
-
debug: opts.debug,
|
|
1393
|
-
recorder: null,
|
|
1394
|
-
transpose: this.#state.embeddingTranspose,
|
|
1395
|
-
activationDtype: 'f32',
|
|
1396
|
-
embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
|
|
1397
|
-
embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
|
|
1398
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1399
|
-
operatorDiagnostics: this.#state.operatorDiagnostics,
|
|
1400
|
-
});
|
|
1401
|
-
|
|
1402
|
-
if (softEmbeddingState) {
|
|
1403
|
-
softEmbeddingStateOwned = softEmbeddingState.releaseOnUse;
|
|
1404
|
-
softEmbeddings = createTensor(
|
|
1405
|
-
softEmbeddingState.buffer,
|
|
1406
|
-
softEmbeddingState.dtype,
|
|
1407
|
-
[canvasLength, hiddenSize],
|
|
1408
|
-
'diffusion_gemma_self_conditioning_soft_embedding'
|
|
1409
|
-
);
|
|
1410
|
-
softEmbeddingStateTensor = softEmbeddings;
|
|
1411
|
-
if (!softEmbeddingState.scaled) {
|
|
1412
|
-
scaledSoftEmbeddings = await runScale(softEmbeddings, Math.sqrt(hiddenSize), {
|
|
1413
|
-
count: elementCount,
|
|
1414
|
-
});
|
|
1415
|
-
if (softEmbeddingStateOwned) {
|
|
1416
|
-
releaseTensorOnce(softEmbeddings);
|
|
1417
|
-
}
|
|
1418
|
-
softEmbeddings = scaledSoftEmbeddings;
|
|
1419
|
-
softEmbeddingStateTensor = null;
|
|
1420
|
-
scaledSoftEmbeddings = null;
|
|
1421
|
-
}
|
|
1422
|
-
} else if (logitsState || logits) {
|
|
1423
|
-
const softmaxTemperature = logitsState?.temperature ?? 1.0;
|
|
1424
|
-
if (logitsState) {
|
|
1425
|
-
softLogitsOwned = logitsState.releaseOnUse;
|
|
1426
|
-
softLogitsTensor = createTensor(
|
|
1427
|
-
logitsState.logitsBuffer,
|
|
1428
|
-
logitsState.logitsDtype,
|
|
1429
|
-
[canvasLength, vocabSize],
|
|
1430
|
-
'diffusion_gemma_self_conditioning_logits'
|
|
1431
|
-
);
|
|
1432
|
-
} else {
|
|
1433
|
-
const logitsBuffer = acquireBuffer(logits.byteLength, undefined, 'diffusion_gemma_self_conditioning_logits');
|
|
1434
|
-
uploadData(logitsBuffer, logits);
|
|
1435
|
-
softLogitsTensor = createTensor(logitsBuffer, 'f32', [canvasLength, vocabSize], 'diffusion_gemma_self_conditioning_logits');
|
|
1436
|
-
}
|
|
1437
|
-
if (canUseChunkedSoftEmbeddingLogits(
|
|
1438
|
-
logitsState,
|
|
1439
|
-
borrowed.softEmbedding?.value,
|
|
1440
|
-
this.#state.embeddingTranspose
|
|
1441
|
-
)) {
|
|
1442
|
-
softEmbeddings = await runSoftEmbeddingLogitsF16(
|
|
1443
|
-
softLogitsTensor,
|
|
1444
|
-
borrowed.softEmbedding.value,
|
|
1445
|
-
canvasLength,
|
|
1446
|
-
hiddenSize,
|
|
1447
|
-
vocabSize,
|
|
1448
|
-
{
|
|
1449
|
-
temperature: softmaxTemperature,
|
|
1450
|
-
chunkRows: resolveDiffusionGemmaSoftEmbeddingChunkRows(this.#state.runtimeConfig),
|
|
1451
|
-
}
|
|
1452
|
-
);
|
|
1453
|
-
} else {
|
|
1454
|
-
softmaxTensor = await runSoftmax(softLogitsTensor, -1, {
|
|
1455
|
-
batchSize: canvasLength,
|
|
1456
|
-
size: vocabSize,
|
|
1457
|
-
temperature: softmaxTemperature,
|
|
1458
|
-
});
|
|
1459
|
-
softEmbeddings = isSplitWeightBuffer(embedBufferRaw)
|
|
1460
|
-
? await runSoftEmbeddingSplitF16(
|
|
1461
|
-
softmaxTensor,
|
|
1462
|
-
embedBufferRaw,
|
|
1463
|
-
canvasLength,
|
|
1464
|
-
hiddenSize,
|
|
1465
|
-
vocabSize
|
|
1466
|
-
)
|
|
1467
|
-
: await runMatmul(
|
|
1468
|
-
softmaxTensor,
|
|
1469
|
-
borrowed.softEmbedding.value,
|
|
1470
|
-
canvasLength,
|
|
1471
|
-
hiddenSize,
|
|
1472
|
-
vocabSize,
|
|
1473
|
-
{
|
|
1474
|
-
transposeB: this.#state.embeddingTranspose === true,
|
|
1475
|
-
role: 'diffusion_gemma_self_conditioning_embed',
|
|
1476
|
-
outputDtype: 'f32',
|
|
1477
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1478
|
-
}
|
|
1479
|
-
);
|
|
1480
|
-
}
|
|
1481
|
-
scaledSoftEmbeddings = await runScale(softEmbeddings, Math.sqrt(hiddenSize), {
|
|
1482
|
-
count: elementCount,
|
|
1483
|
-
});
|
|
1484
|
-
if (softEmbeddings !== softEmbeddingStateTensor || softEmbeddingStateOwned) {
|
|
1485
|
-
releaseTensorOnce(softEmbeddings);
|
|
1486
|
-
}
|
|
1487
|
-
softEmbeddings = scaledSoftEmbeddings;
|
|
1488
|
-
scaledSoftEmbeddings = null;
|
|
1489
|
-
} else {
|
|
1490
|
-
const zeroBuffer = acquireBuffer(elementCount * Float32Array.BYTES_PER_ELEMENT, undefined, 'diffusion_gemma_self_conditioning_zero');
|
|
1491
|
-
uploadData(zeroBuffer, new Uint8Array(elementCount * Float32Array.BYTES_PER_ELEMENT));
|
|
1492
|
-
softEmbeddings = createTensor(zeroBuffer, 'f32', [canvasLength, hiddenSize], 'diffusion_gemma_self_conditioning_zero');
|
|
1493
|
-
}
|
|
1494
|
-
|
|
1495
|
-
normedSoft = await runRMSNorm(softEmbeddings, borrowed.preNorm.value, config.rmsNormEps, {
|
|
1496
|
-
batchSize: canvasLength,
|
|
1497
|
-
hiddenSize,
|
|
1498
|
-
rmsNormWeightOffset: false,
|
|
1499
|
-
});
|
|
1500
|
-
const intermediateSize = config.intermediateSize;
|
|
1501
|
-
gate = await runMatmul(
|
|
1502
|
-
normedSoft,
|
|
1503
|
-
borrowed.gateProj.value,
|
|
1504
|
-
canvasLength,
|
|
1505
|
-
intermediateSize,
|
|
1506
|
-
hiddenSize,
|
|
1507
|
-
{
|
|
1508
|
-
transposeB: 'auto',
|
|
1509
|
-
role: 'diffusion_gemma_self_conditioning_gate',
|
|
1510
|
-
outputDtype: 'f32',
|
|
1511
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1512
|
-
}
|
|
1513
|
-
);
|
|
1514
|
-
up = await runMatmul(
|
|
1515
|
-
normedSoft,
|
|
1516
|
-
borrowed.upProj.value,
|
|
1517
|
-
canvasLength,
|
|
1518
|
-
intermediateSize,
|
|
1519
|
-
hiddenSize,
|
|
1520
|
-
{
|
|
1521
|
-
transposeB: 'auto',
|
|
1522
|
-
role: 'diffusion_gemma_self_conditioning_up',
|
|
1523
|
-
outputDtype: 'f32',
|
|
1524
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1525
|
-
}
|
|
1526
|
-
);
|
|
1527
|
-
activated = await runGeLU(up, {
|
|
1528
|
-
size: canvasLength * intermediateSize,
|
|
1529
|
-
gate,
|
|
1530
|
-
});
|
|
1531
|
-
down = await runMatmul(
|
|
1532
|
-
activated,
|
|
1533
|
-
borrowed.downProj.value,
|
|
1534
|
-
canvasLength,
|
|
1535
|
-
hiddenSize,
|
|
1536
|
-
intermediateSize,
|
|
1537
|
-
{
|
|
1538
|
-
transposeB: 'auto',
|
|
1539
|
-
role: 'diffusion_gemma_self_conditioning_down',
|
|
1540
|
-
outputDtype: 'f32',
|
|
1541
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1542
|
-
}
|
|
1543
|
-
);
|
|
1544
|
-
combined = await runResidualAdd(baseEmbeddings, down, elementCount, {
|
|
1545
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1546
|
-
});
|
|
1547
|
-
const postNormWeight = weights.postNorm
|
|
1548
|
-
? borrowNormWeight(weights.postNorm, 'diffusion_gemma_self_conditioning.post_norm')
|
|
1549
|
-
: {
|
|
1550
|
-
value: getQKNormOnesBuffer(hiddenSize),
|
|
1551
|
-
owned: false,
|
|
1552
|
-
};
|
|
1553
|
-
borrowed.postNorm = postNormWeight;
|
|
1554
|
-
output = await runRMSNorm(combined, postNormWeight.value, config.rmsNormEps, {
|
|
1555
|
-
batchSize: canvasLength,
|
|
1556
|
-
hiddenSize,
|
|
1557
|
-
rmsNormWeightOffset: false,
|
|
1558
|
-
});
|
|
1559
|
-
return output;
|
|
1560
|
-
} catch (error) {
|
|
1561
|
-
releaseTensorOnce(output);
|
|
1562
|
-
throw error;
|
|
1563
|
-
} finally {
|
|
1564
|
-
releaseTensorOnce(combined);
|
|
1565
|
-
releaseTensorOnce(down);
|
|
1566
|
-
releaseTensorOnce(activated);
|
|
1567
|
-
releaseTensorOnce(up);
|
|
1568
|
-
releaseTensorOnce(gate);
|
|
1569
|
-
releaseTensorOnce(normedSoft);
|
|
1570
|
-
if (softEmbeddings !== softEmbeddingStateTensor || softEmbeddingStateOwned) {
|
|
1571
|
-
releaseTensorOnce(softEmbeddings);
|
|
1572
|
-
}
|
|
1573
|
-
releaseTensorOnce(softmaxTensor);
|
|
1574
|
-
if (softLogitsOwned) {
|
|
1575
|
-
releaseTensorOnce(softLogitsTensor);
|
|
1576
|
-
}
|
|
1577
|
-
releaseTensorOnce(baseEmbeddings);
|
|
1578
|
-
releaseBorrowedWeight(borrowed.postNorm);
|
|
1579
|
-
releaseBorrowedWeight(borrowed.downProj);
|
|
1580
|
-
releaseBorrowedWeight(borrowed.upProj);
|
|
1581
|
-
releaseBorrowedWeight(borrowed.gateProj);
|
|
1582
|
-
releaseBorrowedWeight(borrowed.preNorm);
|
|
1583
|
-
releaseBorrowedWeight(borrowed.softEmbedding);
|
|
1584
|
-
}
|
|
513
|
+
return _createDiffusionGemmaSelfConditioningEmbeddingsImpl.apply(this, arguments);
|
|
1585
514
|
}
|
|
1586
515
|
|
|
1587
516
|
async _createDiffusionGemmaSelfConditioningSoftEmbeddingState(logitsState, canvasLength, hiddenSize, vocabSize) {
|
|
1588
|
-
|
|
1589
|
-
if (!canUseChunkedSoftEmbeddingLogits(
|
|
1590
|
-
logitsState,
|
|
1591
|
-
embedBufferRaw,
|
|
1592
|
-
this.#state.embeddingTranspose
|
|
1593
|
-
)) {
|
|
1594
|
-
return null;
|
|
1595
|
-
}
|
|
1596
|
-
|
|
1597
|
-
const borrowed = borrowLinearWeight(embedBufferRaw, 'diffusion_gemma_self_conditioning.embed_tokens');
|
|
1598
|
-
let softEmbedding = null;
|
|
1599
|
-
let scaledSoftEmbedding = null;
|
|
1600
|
-
try {
|
|
1601
|
-
const logitsTensor = createTensor(
|
|
1602
|
-
logitsState.logitsBuffer,
|
|
1603
|
-
logitsState.logitsDtype,
|
|
1604
|
-
[canvasLength, vocabSize],
|
|
1605
|
-
'diffusion_gemma_self_conditioning_logits'
|
|
1606
|
-
);
|
|
1607
|
-
softEmbedding = await runSoftEmbeddingLogitsF16(
|
|
1608
|
-
logitsTensor,
|
|
1609
|
-
borrowed.value,
|
|
1610
|
-
canvasLength,
|
|
1611
|
-
hiddenSize,
|
|
1612
|
-
vocabSize,
|
|
1613
|
-
{
|
|
1614
|
-
temperature: logitsState.temperature,
|
|
1615
|
-
chunkRows: resolveDiffusionGemmaSoftEmbeddingChunkRows(this.#state.runtimeConfig),
|
|
1616
|
-
}
|
|
1617
|
-
);
|
|
1618
|
-
scaledSoftEmbedding = await runScale(softEmbedding, Math.sqrt(hiddenSize), {
|
|
1619
|
-
count: canvasLength * hiddenSize,
|
|
1620
|
-
});
|
|
1621
|
-
const returnedBuffer = scaledSoftEmbedding.buffer;
|
|
1622
|
-
return {
|
|
1623
|
-
kind: 'soft_embedding',
|
|
1624
|
-
buffer: returnedBuffer,
|
|
1625
|
-
dtype: scaledSoftEmbedding.dtype,
|
|
1626
|
-
canvasLength,
|
|
1627
|
-
hiddenSize,
|
|
1628
|
-
scaled: true,
|
|
1629
|
-
releaseOnUse: true,
|
|
1630
|
-
release() {
|
|
1631
|
-
releaseBuffer(returnedBuffer);
|
|
1632
|
-
},
|
|
1633
|
-
};
|
|
1634
|
-
} finally {
|
|
1635
|
-
if (softEmbedding?.buffer && softEmbedding.buffer !== scaledSoftEmbedding?.buffer) {
|
|
1636
|
-
releaseBuffer(softEmbedding.buffer);
|
|
1637
|
-
}
|
|
1638
|
-
releaseBorrowedWeight(borrowed);
|
|
1639
|
-
}
|
|
517
|
+
return _createDiffusionGemmaSelfConditioningSoftEmbeddingStateImpl.apply(this, arguments);
|
|
1640
518
|
}
|
|
1641
519
|
|
|
1642
520
|
async computeDiffusionGemmaCanvasLogits(args, options = {}) {
|
|
1643
|
-
|
|
1644
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
1645
|
-
throw new Error('Generation already in progress');
|
|
1646
|
-
}
|
|
1647
|
-
const canvasIds = normalizeCanvasTokenIds(args?.canvas, 'computeDiffusionGemmaCanvasLogits');
|
|
1648
|
-
if (canvasIds.length === 0) {
|
|
1649
|
-
throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasLogits requires at least one canvas token.');
|
|
1650
|
-
}
|
|
1651
|
-
this._assertTokenIdsInRange(canvasIds, 'computeDiffusionGemmaCanvasLogits.canvas');
|
|
1652
|
-
const seqLenBefore = this.#state.currentSeqLen;
|
|
1653
|
-
const opts = {
|
|
1654
|
-
...resolvePrefillOptions(this.#state, {
|
|
1655
|
-
...options,
|
|
1656
|
-
useChatTemplate: false,
|
|
1657
|
-
}),
|
|
1658
|
-
_diffusionGemmaDecoder: true,
|
|
1659
|
-
};
|
|
1660
|
-
let selfConditioned = null;
|
|
1661
|
-
let currentHiddenBuffer = null;
|
|
1662
|
-
try {
|
|
1663
|
-
selfConditioned = await this._createDiffusionGemmaSelfConditioningEmbeddings(
|
|
1664
|
-
canvasIds,
|
|
1665
|
-
args?.selfConditioningLogits ?? null,
|
|
1666
|
-
opts
|
|
1667
|
-
);
|
|
1668
|
-
const prefillResult = await this._prefillToHidden(canvasIds, {
|
|
1669
|
-
...opts,
|
|
1670
|
-
embeddingOverrides: {
|
|
1671
|
-
offset: 0,
|
|
1672
|
-
prefixLength: canvasIds.length,
|
|
1673
|
-
embeddings: selfConditioned.buffer,
|
|
1674
|
-
},
|
|
1675
|
-
});
|
|
1676
|
-
const {
|
|
1677
|
-
numTokens,
|
|
1678
|
-
currentRecorder,
|
|
1679
|
-
recordProfile,
|
|
1680
|
-
debugCheckBuffer,
|
|
1681
|
-
} = prefillResult;
|
|
1682
|
-
currentHiddenBuffer = prefillResult.currentHiddenBuffer;
|
|
1683
|
-
if (currentRecorder) {
|
|
1684
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
1685
|
-
await currentRecorder.submitAndWait();
|
|
1686
|
-
await recordProfile(currentRecorder);
|
|
1687
|
-
} else {
|
|
1688
|
-
const device = getDevice();
|
|
1689
|
-
if (device) {
|
|
1690
|
-
await device.queue.onSubmittedWorkDone();
|
|
1691
|
-
}
|
|
1692
|
-
}
|
|
1693
|
-
const logits = await computeLogits(
|
|
1694
|
-
currentHiddenBuffer,
|
|
1695
|
-
numTokens,
|
|
1696
|
-
getLogitsWeights(this.#state),
|
|
1697
|
-
getLogitsConfig(this.#state),
|
|
1698
|
-
this.#state.useGPU,
|
|
1699
|
-
this.#state.debugFlags,
|
|
1700
|
-
undefined,
|
|
1701
|
-
debugCheckBuffer,
|
|
1702
|
-
this.#state.runtimeConfig.shared.debug.probes,
|
|
1703
|
-
{ lastPositionOnly: false },
|
|
1704
|
-
this.#state.operatorDiagnostics
|
|
1705
|
-
);
|
|
1706
|
-
const expected = canvasIds.length * this.#state.modelConfig.vocabSize;
|
|
1707
|
-
if (logits.length !== expected) {
|
|
1708
|
-
throw new Error(
|
|
1709
|
-
`[DiffusionGemma] canvas logits length mismatch: expected ${expected}, got ${logits.length}.`
|
|
1710
|
-
);
|
|
1711
|
-
}
|
|
1712
|
-
return logits;
|
|
1713
|
-
} finally {
|
|
1714
|
-
this.#state.currentSeqLen = seqLenBefore;
|
|
1715
|
-
if (currentHiddenBuffer) {
|
|
1716
|
-
releaseBuffer(currentHiddenBuffer);
|
|
1717
|
-
}
|
|
1718
|
-
if (selfConditioned?.buffer) {
|
|
1719
|
-
releaseBuffer(selfConditioned.buffer);
|
|
1720
|
-
}
|
|
1721
|
-
}
|
|
521
|
+
return computeDiffusionGemmaCanvasLogitsImpl.apply(this, arguments);
|
|
1722
522
|
}
|
|
1723
523
|
|
|
1724
524
|
async computeDiffusionGemmaCanvasStep(args, options = {}) {
|
|
1725
|
-
|
|
1726
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
1727
|
-
throw new Error('Generation already in progress');
|
|
1728
|
-
}
|
|
1729
|
-
const canvasIds = normalizeCanvasTokenIds(args?.canvas, 'computeDiffusionGemmaCanvasStep');
|
|
1730
|
-
if (canvasIds.length === 0) {
|
|
1731
|
-
throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasStep requires at least one canvas token.');
|
|
1732
|
-
}
|
|
1733
|
-
this._assertTokenIdsInRange(canvasIds, 'computeDiffusionGemmaCanvasStep.canvas');
|
|
1734
|
-
const temperature = args?.temperature;
|
|
1735
|
-
if (typeof temperature !== 'number' || !Number.isFinite(temperature) || temperature <= 0) {
|
|
1736
|
-
throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasStep requires a positive temperature.');
|
|
1737
|
-
}
|
|
1738
|
-
const seqLenBefore = this.#state.currentSeqLen;
|
|
1739
|
-
const opts = {
|
|
1740
|
-
...resolvePrefillOptions(this.#state, {
|
|
1741
|
-
...options,
|
|
1742
|
-
useChatTemplate: false,
|
|
1743
|
-
}),
|
|
1744
|
-
_diffusionGemmaDecoder: true,
|
|
1745
|
-
};
|
|
1746
|
-
let selfConditioned = null;
|
|
1747
|
-
let currentHiddenBuffer = null;
|
|
1748
|
-
let logitsBuffer = null;
|
|
1749
|
-
let statsBuffers = null;
|
|
1750
|
-
try {
|
|
1751
|
-
selfConditioned = await this._createDiffusionGemmaSelfConditioningEmbeddings(
|
|
1752
|
-
canvasIds,
|
|
1753
|
-
args?.selfConditioningLogits ?? null,
|
|
1754
|
-
opts
|
|
1755
|
-
);
|
|
1756
|
-
const prefillResult = await this._prefillToHidden(canvasIds, {
|
|
1757
|
-
...opts,
|
|
1758
|
-
embeddingOverrides: {
|
|
1759
|
-
offset: 0,
|
|
1760
|
-
prefixLength: canvasIds.length,
|
|
1761
|
-
embeddings: selfConditioned.buffer,
|
|
1762
|
-
},
|
|
1763
|
-
});
|
|
1764
|
-
const {
|
|
1765
|
-
numTokens,
|
|
1766
|
-
currentRecorder,
|
|
1767
|
-
recordProfile,
|
|
1768
|
-
debugCheckBuffer,
|
|
1769
|
-
} = prefillResult;
|
|
1770
|
-
currentHiddenBuffer = prefillResult.currentHiddenBuffer;
|
|
1771
|
-
if (currentRecorder) {
|
|
1772
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
1773
|
-
await currentRecorder.submitAndWait();
|
|
1774
|
-
await recordProfile(currentRecorder);
|
|
1775
|
-
} else {
|
|
1776
|
-
const device = getDevice();
|
|
1777
|
-
if (device) {
|
|
1778
|
-
await device.queue.onSubmittedWorkDone();
|
|
1779
|
-
}
|
|
1780
|
-
}
|
|
1781
|
-
if (selfConditioned?.buffer) {
|
|
1782
|
-
releaseBuffer(selfConditioned.buffer);
|
|
1783
|
-
selfConditioned = null;
|
|
1784
|
-
}
|
|
1785
|
-
|
|
1786
|
-
const logitsResult = await computeLogitsGPU(
|
|
1787
|
-
currentHiddenBuffer,
|
|
1788
|
-
numTokens,
|
|
1789
|
-
getLogitsWeights(this.#state),
|
|
1790
|
-
getLogitsConfig(this.#state),
|
|
1791
|
-
this.#state.debugFlags,
|
|
1792
|
-
this.#state.operatorDiagnostics
|
|
1793
|
-
);
|
|
1794
|
-
if (!logitsResult?.logitsBuffer) {
|
|
1795
|
-
throw new Error('[DiffusionGemma] GPU canvas step requires GPU logits.');
|
|
1796
|
-
}
|
|
1797
|
-
logitsBuffer = logitsResult.logitsBuffer;
|
|
1798
|
-
if (currentHiddenBuffer) {
|
|
1799
|
-
releaseBuffer(currentHiddenBuffer);
|
|
1800
|
-
currentHiddenBuffer = null;
|
|
1801
|
-
}
|
|
1802
|
-
if (logitsResult.logitsDtype !== 'f32') {
|
|
1803
|
-
throw new Error(
|
|
1804
|
-
`[DiffusionGemma] GPU canvas stats require f32 logits, got "${logitsResult.logitsDtype}".`
|
|
1805
|
-
);
|
|
1806
|
-
}
|
|
1807
|
-
if (logitsResult.vocabSize !== this.#state.modelConfig.vocabSize) {
|
|
1808
|
-
throw new Error(
|
|
1809
|
-
`[DiffusionGemma] canvas logits vocab mismatch: expected ${this.#state.modelConfig.vocabSize}, ` +
|
|
1810
|
-
`got ${logitsResult.vocabSize}.`
|
|
1811
|
-
);
|
|
1812
|
-
}
|
|
1813
|
-
|
|
1814
|
-
statsBuffers = await runDiffusionGemmaCanvasStats(logitsBuffer, {
|
|
1815
|
-
canvasLength: canvasIds.length,
|
|
1816
|
-
vocabSize: logitsResult.vocabSize,
|
|
1817
|
-
temperature,
|
|
1818
|
-
padTokenId: this.#state.modelConfig.diffusionGemma?.padTokenId ?? null,
|
|
1819
|
-
logitSoftcap: this.#state.modelConfig.finalLogitSoftcapping ?? 0,
|
|
1820
|
-
});
|
|
1821
|
-
const [argmaxData, entropyData] = await Promise.all([
|
|
1822
|
-
readBuffer(statsBuffers.argmaxBuffer, canvasIds.length * Uint32Array.BYTES_PER_ELEMENT),
|
|
1823
|
-
readBuffer(statsBuffers.entropyBuffer, canvasIds.length * Float32Array.BYTES_PER_ELEMENT),
|
|
1824
|
-
]);
|
|
1825
|
-
const argmaxCanvas = Int32Array.from(new Uint32Array(argmaxData));
|
|
1826
|
-
const entropies = new Float32Array(entropyData);
|
|
1827
|
-
const logitsState = {
|
|
1828
|
-
logitsBuffer,
|
|
1829
|
-
logitsDtype: logitsResult.logitsDtype,
|
|
1830
|
-
vocabSize: logitsResult.vocabSize,
|
|
1831
|
-
canvasLength: canvasIds.length,
|
|
1832
|
-
temperature,
|
|
1833
|
-
releaseOnUse: true,
|
|
1834
|
-
};
|
|
1835
|
-
const selfConditioningState =
|
|
1836
|
-
await this._createDiffusionGemmaSelfConditioningSoftEmbeddingState(
|
|
1837
|
-
logitsState,
|
|
1838
|
-
canvasIds.length,
|
|
1839
|
-
this.#state.modelConfig.hiddenSize,
|
|
1840
|
-
logitsResult.vocabSize
|
|
1841
|
-
);
|
|
1842
|
-
if (selfConditioningState) {
|
|
1843
|
-
releaseBuffer(logitsBuffer);
|
|
1844
|
-
logitsBuffer = null;
|
|
1845
|
-
} else {
|
|
1846
|
-
const returnedLogitsBuffer = logitsBuffer;
|
|
1847
|
-
logitsBuffer = null;
|
|
1848
|
-
logitsState.release = () => {
|
|
1849
|
-
releaseBuffer(returnedLogitsBuffer);
|
|
1850
|
-
};
|
|
1851
|
-
}
|
|
1852
|
-
return {
|
|
1853
|
-
argmaxCanvas,
|
|
1854
|
-
entropies,
|
|
1855
|
-
selfConditioningLogits: selfConditioningState ?? logitsState,
|
|
1856
|
-
};
|
|
1857
|
-
} finally {
|
|
1858
|
-
this.#state.currentSeqLen = seqLenBefore;
|
|
1859
|
-
if (statsBuffers?.argmaxBuffer) {
|
|
1860
|
-
releaseBuffer(statsBuffers.argmaxBuffer);
|
|
1861
|
-
}
|
|
1862
|
-
if (statsBuffers?.entropyBuffer) {
|
|
1863
|
-
releaseBuffer(statsBuffers.entropyBuffer);
|
|
1864
|
-
}
|
|
1865
|
-
if (logitsBuffer) {
|
|
1866
|
-
releaseBuffer(logitsBuffer);
|
|
1867
|
-
}
|
|
1868
|
-
if (currentHiddenBuffer) {
|
|
1869
|
-
releaseBuffer(currentHiddenBuffer);
|
|
1870
|
-
}
|
|
1871
|
-
if (selfConditioned?.buffer) {
|
|
1872
|
-
releaseBuffer(selfConditioned.buffer);
|
|
1873
|
-
}
|
|
1874
|
-
}
|
|
525
|
+
return computeDiffusionGemmaCanvasStepImpl.apply(this, arguments);
|
|
1875
526
|
}
|
|
1876
527
|
|
|
1877
528
|
async *generate(prompt, options = {}) {
|
|
@@ -1883,873 +534,35 @@ export class PipelineGenerator {
|
|
|
1883
534
|
}
|
|
1884
535
|
|
|
1885
536
|
async generateTokenIds(prompt, options = {}) {
|
|
1886
|
-
|
|
1887
|
-
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
1888
|
-
|
|
1889
|
-
validateCallTimeOptions(options);
|
|
1890
|
-
|
|
1891
|
-
this.#state.isGenerating = true;
|
|
1892
|
-
this._resetDecodeRuntimeState();
|
|
1893
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
1894
|
-
this.#state.stats.gpuTimeDecodeMs = undefined;
|
|
1895
|
-
this.#state.stats.decodeRecordMs = 0;
|
|
1896
|
-
this.#state.stats.decodeRecordOps = 0;
|
|
1897
|
-
this.#state.stats.decodeRecordPasses = 0;
|
|
1898
|
-
this.#state.stats.decodeRecordOpLabels = {};
|
|
1899
|
-
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
1900
|
-
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
1901
|
-
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
1902
|
-
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
1903
|
-
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
1904
|
-
this.#state.stats.prefillRecordMs = 0;
|
|
1905
|
-
this.#state.stats.prefillRecordOps = 0;
|
|
1906
|
-
this.#state.stats.prefillRecordPasses = 0;
|
|
1907
|
-
this.#state.stats.prefillRecordOpLabels = {};
|
|
1908
|
-
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
1909
|
-
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
1910
|
-
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
1911
|
-
this.#state.stats.singleTokenReadbackMapWaitMs = 0;
|
|
1912
|
-
this.#state.stats.singleTokenReadbackCleanupMs = 0;
|
|
1913
|
-
this.#state.stats.singleTokenReadbackCopyMs = 0;
|
|
1914
|
-
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
1915
|
-
this.#state.stats.pleHotVocabularyHits = 0;
|
|
1916
|
-
this.#state.stats.pleHotVocabularyMisses = 0;
|
|
1917
|
-
this.#state.stats.plePreparedTokenCacheHits = 0;
|
|
1918
|
-
this.#state.stats.plePreparedTokenCacheMisses = 0;
|
|
1919
|
-
this.#state.stats.plePreparedTokenCacheEntries = 0;
|
|
1920
|
-
this.#state.stats.plePreparedTokenCacheBytes = 0;
|
|
1921
|
-
this.#state.stats.decodeMode = null;
|
|
1922
|
-
this.#state.stats.batchGuardReason = null;
|
|
1923
|
-
this.#state.stats.stopReason = null;
|
|
1924
|
-
this.#state.stats.stopTokenId = null;
|
|
1925
|
-
this.#state.stats.ttftMs = 0;
|
|
1926
|
-
const startTime = performance.now();
|
|
1927
|
-
const opts = resolveGenerateOptions(this.#state, options);
|
|
1928
|
-
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
1929
|
-
// Validate and normalize sampling parameters through single source of truth
|
|
1930
|
-
const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
|
|
1931
|
-
opts.temperature = samplingConfig.temperature;
|
|
1932
|
-
opts.topP = samplingConfig.topP;
|
|
1933
|
-
opts.topK = samplingConfig.topK;
|
|
1934
|
-
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
1935
|
-
opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this.#state, samplingConfig);
|
|
1936
|
-
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
1937
|
-
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
1938
|
-
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
1939
|
-
if (diagnosticsEnabled || tsirFixtureCfg) {
|
|
1940
|
-
const captureConfig = {
|
|
1941
|
-
...createDefaultCaptureConfig(),
|
|
1942
|
-
enabled: true,
|
|
1943
|
-
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
1944
|
-
...(options?.diagnostics?.captureConfig ?? {}),
|
|
1945
|
-
};
|
|
1946
|
-
validateCaptureConfig(captureConfig);
|
|
1947
|
-
this.#state.operatorDiagnostics = {
|
|
1948
|
-
enabled: diagnosticsEnabled === true,
|
|
1949
|
-
captureConfig,
|
|
1950
|
-
emitter: diagnosticsEnabled ? new OperatorEventEmitter({
|
|
1951
|
-
modelHash: this.#state.manifest?.modelId ?? null,
|
|
1952
|
-
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
1953
|
-
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
1954
|
-
}) : null,
|
|
1955
|
-
tsirFixture: tsirFixtureCfg ? {
|
|
1956
|
-
dir: tsirFixtureCfg.dir,
|
|
1957
|
-
layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
|
|
1958
|
-
? tsirFixtureCfg.layerFilter
|
|
1959
|
-
: null,
|
|
1960
|
-
records: [],
|
|
1961
|
-
} : null,
|
|
1962
|
-
};
|
|
1963
|
-
}
|
|
1964
|
-
|
|
1965
|
-
try {
|
|
1966
|
-
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
1967
|
-
const prefillStart = performance.now();
|
|
1968
|
-
const { inputIds, logits: initialPrefillLogits } = await this._prefillPromptToLogits(prompt, opts, 'generateTokenIds');
|
|
1969
|
-
let prefillLogits = initialPrefillLogits;
|
|
1970
|
-
this.#state.stats.prefillTimeMs = performance.now() - prefillStart;
|
|
1971
|
-
this._assertTokenIdsInRange(inputIds, 'generateTokenIds.prefillTokens');
|
|
1972
|
-
const generatedIds = [...inputIds];
|
|
1973
|
-
this.#state.stats.prefillTokens = inputIds.length;
|
|
1974
|
-
|
|
1975
|
-
let firstToken;
|
|
1976
|
-
try {
|
|
1977
|
-
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
1978
|
-
} catch (error) {
|
|
1979
|
-
if (!this._shouldUseFinitenessFallback(error, 'prefill-sample')) {
|
|
1980
|
-
throw error;
|
|
1981
|
-
}
|
|
1982
|
-
prefillLogits = await this._retryWithPersistentFinitenessFallback(
|
|
1983
|
-
opts,
|
|
1984
|
-
'prefill-sample',
|
|
1985
|
-
opts.maxTokens,
|
|
1986
|
-
() => this._prefill(inputIds, opts),
|
|
1987
|
-
prefillStartSeqLen
|
|
1988
|
-
);
|
|
1989
|
-
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
1990
|
-
}
|
|
1991
|
-
|
|
1992
|
-
const stopTokenIds = this.#state.modelConfig.stopTokenIds;
|
|
1993
|
-
const eosToken = this.#state.tokenizer.getSpecialTokens?.()?.eos;
|
|
1994
|
-
const stopSequenceStart = inputIds.length;
|
|
1995
|
-
generatedIds.push(firstToken);
|
|
1996
|
-
const tokenIds = [firstToken];
|
|
1997
|
-
this.#state.stats.ttftMs = performance.now() - startTime;
|
|
1998
|
-
markKernelCacheWarmed();
|
|
1999
|
-
|
|
2000
|
-
const decodeRuntime = {
|
|
2001
|
-
stopTokenIds,
|
|
2002
|
-
eosToken,
|
|
2003
|
-
stopSequenceStart,
|
|
2004
|
-
decodeToken: () => '',
|
|
2005
|
-
emitMode: 'token',
|
|
2006
|
-
};
|
|
2007
|
-
|
|
2008
|
-
if (!this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
|
|
2009
|
-
for await (const tokenId of this._runDecodeLoop(generatedIds, opts, options, decodeRuntime)) {
|
|
2010
|
-
tokenIds.push(tokenId);
|
|
2011
|
-
}
|
|
2012
|
-
} else {
|
|
2013
|
-
this._recordStopReason('stop-token-or-sequence', firstToken);
|
|
2014
|
-
this.#state.stats.decodeTimeMs = 0;
|
|
2015
|
-
this.#state.stats.tokensGenerated = 1;
|
|
2016
|
-
this.#state.stats.decodeTokens = 1;
|
|
2017
|
-
}
|
|
2018
|
-
|
|
2019
|
-
this.#state.stats.totalTimeMs = performance.now() - startTime;
|
|
2020
|
-
|
|
2021
|
-
return {
|
|
2022
|
-
tokenIds,
|
|
2023
|
-
stats: this.#state.stats,
|
|
2024
|
-
};
|
|
2025
|
-
} finally {
|
|
2026
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
2027
|
-
if (this.#state.stats.executionPlan) {
|
|
2028
|
-
this.#state.stats.executionPlan.finalActivePlanId = this.#state.executionPlanState?.activePlanId ?? null;
|
|
2029
|
-
}
|
|
2030
|
-
resetActiveExecutionPlan(this.#state);
|
|
2031
|
-
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
2032
|
-
? {
|
|
2033
|
-
enabled: true,
|
|
2034
|
-
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
2035
|
-
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
2036
|
-
}
|
|
2037
|
-
: null;
|
|
2038
|
-
this.#state.operatorDiagnostics = null;
|
|
2039
|
-
this.#state.isGenerating = false;
|
|
2040
|
-
}
|
|
537
|
+
return generateTokenIdsImpl.apply(this, arguments);
|
|
2041
538
|
}
|
|
2042
539
|
|
|
2043
|
-
|
|
2044
540
|
async prefillKVOnly(prompt, options = {}) {
|
|
2045
|
-
|
|
2046
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2047
|
-
throw new Error('Generation already in progress');
|
|
2048
|
-
}
|
|
2049
|
-
assertIncrementalDecodeSupport(this.#state, 'prefillKVOnly');
|
|
2050
|
-
this._resetDecodeRuntimeState();
|
|
2051
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2052
|
-
this.#state.stats.prefillProfileSteps = [];
|
|
2053
|
-
const opts = resolvePrefillOptions(this.#state, options);
|
|
2054
|
-
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
2055
|
-
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillKVOnly', opts.inputIds);
|
|
2056
|
-
if (opts.debug) {
|
|
2057
|
-
log.debug('Pipeline', `PrefillKVOnly: ${inputIds.length} tokens`);
|
|
2058
|
-
}
|
|
2059
|
-
|
|
2060
|
-
try {
|
|
2061
|
-
let prefillResult;
|
|
2062
|
-
try {
|
|
2063
|
-
prefillResult = await this._prefillToHidden(inputIds, opts);
|
|
2064
|
-
} catch (error) {
|
|
2065
|
-
if (this._shouldUseFinitenessFallback(error, 'prefillKVOnly')) {
|
|
2066
|
-
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillKVOnly. Retrying with F32 precision.`);
|
|
2067
|
-
prefillResult = await this._retryWithPersistentFinitenessFallback(
|
|
2068
|
-
opts,
|
|
2069
|
-
'prefillKVOnly',
|
|
2070
|
-
1,
|
|
2071
|
-
() => this._prefillToHidden(inputIds, opts),
|
|
2072
|
-
prefillStartSeqLen
|
|
2073
|
-
);
|
|
2074
|
-
} else {
|
|
2075
|
-
throw error;
|
|
2076
|
-
}
|
|
2077
|
-
}
|
|
2078
|
-
|
|
2079
|
-
const {
|
|
2080
|
-
numTokens,
|
|
2081
|
-
startPos,
|
|
2082
|
-
currentRecorder,
|
|
2083
|
-
recordProfile,
|
|
2084
|
-
currentHiddenBuffer,
|
|
2085
|
-
} = prefillResult;
|
|
2086
|
-
|
|
2087
|
-
// Ensure prefill work completes before returning a usable snapshot.
|
|
2088
|
-
if (currentRecorder) {
|
|
2089
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
2090
|
-
await currentRecorder.submitAndWait();
|
|
2091
|
-
await recordProfile(currentRecorder);
|
|
2092
|
-
} else {
|
|
2093
|
-
const device = getDevice();
|
|
2094
|
-
if (device) {
|
|
2095
|
-
await device.queue.onSubmittedWorkDone();
|
|
2096
|
-
}
|
|
2097
|
-
}
|
|
2098
|
-
|
|
2099
|
-
this.#state.currentSeqLen = startPos + numTokens;
|
|
2100
|
-
releaseBuffer(currentHiddenBuffer);
|
|
2101
|
-
|
|
2102
|
-
const snapshot = this.#state.kvCache?.clone();
|
|
2103
|
-
if (!snapshot) {
|
|
2104
|
-
throw new Error('KV cache unavailable after prefill');
|
|
2105
|
-
}
|
|
2106
|
-
|
|
2107
|
-
return {
|
|
2108
|
-
cache: snapshot,
|
|
2109
|
-
seqLen: this.#state.currentSeqLen,
|
|
2110
|
-
tokens: inputIds,
|
|
2111
|
-
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
2112
|
-
};
|
|
2113
|
-
} finally {
|
|
2114
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
2115
|
-
}
|
|
541
|
+
return prefillKVOnlyImpl.apply(this, arguments);
|
|
2116
542
|
}
|
|
2117
543
|
|
|
2118
544
|
async prefillForLoRATraining(inputIds, options = {}) {
|
|
2119
|
-
|
|
2120
|
-
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
2121
|
-
if (!Array.isArray(inputIds) && !(inputIds instanceof Int32Array) && !(inputIds instanceof Uint32Array)) {
|
|
2122
|
-
throw new Error('prefillForLoRATraining requires token IDs.');
|
|
2123
|
-
}
|
|
2124
|
-
const tokens = Array.from(inputIds, (value) => Number(value));
|
|
2125
|
-
if (tokens.length === 0) throw new Error('prefillForLoRATraining requires at least one token.');
|
|
2126
|
-
this._assertTokenIdsInRange(tokens, 'prefillForLoRATraining.inputIds');
|
|
2127
|
-
|
|
2128
|
-
const config = this.#state.modelConfig;
|
|
2129
|
-
const layerIdx = Number(options.layerIdx);
|
|
2130
|
-
if (!Number.isInteger(layerIdx) || layerIdx !== config.numLayers - 1) {
|
|
2131
|
-
throw new Error(`native_lora_target_not_supported: layerIdx must be the final layer (${config.numLayers - 1}).`);
|
|
2132
|
-
}
|
|
2133
|
-
if (options.module !== 'down_proj') {
|
|
2134
|
-
throw new Error('native_lora_target_not_supported: module must be down_proj.');
|
|
2135
|
-
}
|
|
2136
|
-
if (config.layerTypes?.[layerIdx] !== 'full_attention') {
|
|
2137
|
-
throw new Error('native_lora_target_not_supported: final layer must be full_attention.');
|
|
2138
|
-
}
|
|
2139
|
-
if (config.postFeedforwardNorm === true) {
|
|
2140
|
-
throw new Error('native_lora_target_not_supported: final-layer post-feedforward normalization is not supported.');
|
|
2141
|
-
}
|
|
2142
|
-
if (this.#state.lora) {
|
|
2143
|
-
throw new Error('native_lora_base_capture_requires_inactive_adapter.');
|
|
2144
|
-
}
|
|
2145
|
-
|
|
2146
|
-
this.resetToSeqLen(0);
|
|
2147
|
-
this._resetDecodeRuntimeState();
|
|
2148
|
-
const opts = resolvePrefillOptions(this.#state, {
|
|
2149
|
-
...options,
|
|
2150
|
-
inputIds: tokens,
|
|
2151
|
-
useChatTemplate: false,
|
|
2152
|
-
});
|
|
2153
|
-
let activationTensor = null;
|
|
2154
|
-
let hiddenTensor = null;
|
|
2155
|
-
let disposed = false;
|
|
2156
|
-
const releaseOwned = () => {
|
|
2157
|
-
if (disposed) return;
|
|
2158
|
-
disposed = true;
|
|
2159
|
-
if (activationTensor?.buffer) releaseBuffer(activationTensor.buffer);
|
|
2160
|
-
if (hiddenTensor?.buffer) releaseBuffer(hiddenTensor.buffer);
|
|
2161
|
-
};
|
|
2162
|
-
const capture = async ({ tensor, numTokens, hiddenSize, recorder }) => {
|
|
2163
|
-
if (activationTensor) {
|
|
2164
|
-
throw new Error('native_lora_training_capture_duplicated.');
|
|
2165
|
-
}
|
|
2166
|
-
const bytesPerElement = tensor.dtype === 'f16' ? 2 : 4;
|
|
2167
|
-
const byteLength = Math.ceil((numTokens * hiddenSize * bytesPerElement) / 4) * 4;
|
|
2168
|
-
const buffer = acquireBuffer(byteLength, undefined, `L${layerIdx}.native_lora_ffn_act`);
|
|
2169
|
-
if (recorder) {
|
|
2170
|
-
recorder.getEncoder().copyBufferToBuffer(tensor.buffer, 0, buffer, 0, byteLength);
|
|
2171
|
-
} else {
|
|
2172
|
-
const device = getDevice();
|
|
2173
|
-
const encoder = device.createCommandEncoder({ label: `L${layerIdx}.native_lora_capture` });
|
|
2174
|
-
encoder.copyBufferToBuffer(tensor.buffer, 0, buffer, 0, byteLength);
|
|
2175
|
-
device.queue.submit([encoder.finish()]);
|
|
2176
|
-
}
|
|
2177
|
-
activationTensor = createTensor(
|
|
2178
|
-
buffer,
|
|
2179
|
-
tensor.dtype,
|
|
2180
|
-
[numTokens, hiddenSize],
|
|
2181
|
-
`L${layerIdx}.native_lora_ffn_act`
|
|
2182
|
-
);
|
|
2183
|
-
};
|
|
2184
|
-
|
|
2185
|
-
try {
|
|
2186
|
-
const prefill = await this._prefillToHidden(tokens, {
|
|
2187
|
-
...opts,
|
|
2188
|
-
_trainingCapture: { layerIdx, stage: 'ffn_act', capture },
|
|
2189
|
-
});
|
|
2190
|
-
if (prefill.currentRecorder) {
|
|
2191
|
-
recordPrefillRecorderStats(this.#state, prefill.currentRecorder);
|
|
2192
|
-
await prefill.currentRecorder.submitAndWait();
|
|
2193
|
-
await prefill.recordProfile(prefill.currentRecorder);
|
|
2194
|
-
} else {
|
|
2195
|
-
await getDevice().queue.onSubmittedWorkDone();
|
|
2196
|
-
}
|
|
2197
|
-
if (!activationTensor) {
|
|
2198
|
-
releaseBuffer(prefill.currentHiddenBuffer);
|
|
2199
|
-
throw new Error('native_lora_training_activation_not_captured.');
|
|
2200
|
-
}
|
|
2201
|
-
hiddenTensor = createTensor(
|
|
2202
|
-
prefill.currentHiddenBuffer,
|
|
2203
|
-
prefill.activationDtype,
|
|
2204
|
-
[prefill.numTokens, config.hiddenSize],
|
|
2205
|
-
`L${layerIdx}.native_lora_base_hidden`
|
|
2206
|
-
);
|
|
2207
|
-
this.resetToSeqLen(0);
|
|
2208
|
-
return {
|
|
2209
|
-
inputIds: tokens,
|
|
2210
|
-
layerIdx,
|
|
2211
|
-
module: 'down_proj',
|
|
2212
|
-
activation: activationTensor,
|
|
2213
|
-
baseHidden: hiddenTensor,
|
|
2214
|
-
dispose: releaseOwned,
|
|
2215
|
-
};
|
|
2216
|
-
} catch (error) {
|
|
2217
|
-
releaseOwned();
|
|
2218
|
-
this.resetToSeqLen(0);
|
|
2219
|
-
throw error;
|
|
2220
|
-
}
|
|
545
|
+
return prefillForLoRATrainingImpl.apply(this, arguments);
|
|
2221
546
|
}
|
|
2222
547
|
|
|
2223
548
|
async prefillWithEmbedding(prompt, options = {}) {
|
|
2224
|
-
|
|
2225
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2226
|
-
throw new Error('Generation already in progress');
|
|
2227
|
-
}
|
|
2228
|
-
assertIncrementalDecodeSupport(this.#state, 'prefillWithEmbedding');
|
|
2229
|
-
const totalStart = performance.now();
|
|
2230
|
-
this._resetDecodeRuntimeState();
|
|
2231
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2232
|
-
this.#state.stats.prefillProfileSteps = [];
|
|
2233
|
-
const opts = resolvePrefillEmbeddingOptions(this.#state, options);
|
|
2234
|
-
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
2235
|
-
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
2236
|
-
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
2237
|
-
if (diagnosticsEnabled || tsirFixtureCfg) {
|
|
2238
|
-
const captureConfig = {
|
|
2239
|
-
...createDefaultCaptureConfig(),
|
|
2240
|
-
enabled: true,
|
|
2241
|
-
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
2242
|
-
...(options?.diagnostics?.captureConfig ?? {}),
|
|
2243
|
-
};
|
|
2244
|
-
validateCaptureConfig(captureConfig);
|
|
2245
|
-
this.#state.operatorDiagnostics = {
|
|
2246
|
-
enabled: diagnosticsEnabled === true,
|
|
2247
|
-
captureConfig,
|
|
2248
|
-
emitter: diagnosticsEnabled ? new OperatorEventEmitter({
|
|
2249
|
-
modelHash: this.#state.manifest?.modelId ?? null,
|
|
2250
|
-
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
2251
|
-
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
2252
|
-
}) : null,
|
|
2253
|
-
tsirFixture: tsirFixtureCfg ? {
|
|
2254
|
-
dir: tsirFixtureCfg.dir,
|
|
2255
|
-
layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
|
|
2256
|
-
? tsirFixtureCfg.layerFilter
|
|
2257
|
-
: null,
|
|
2258
|
-
records: [],
|
|
2259
|
-
} : null,
|
|
2260
|
-
};
|
|
2261
|
-
}
|
|
2262
|
-
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
2263
|
-
const inputStart = performance.now();
|
|
2264
|
-
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
|
|
2265
|
-
const inputMs = performance.now() - inputStart;
|
|
2266
|
-
if (opts.debug) {
|
|
2267
|
-
log.debug('Pipeline', `PrefillWithEmbedding: ${inputIds.length} tokens (mode=${opts.embeddingMode})`);
|
|
2268
|
-
}
|
|
2269
|
-
|
|
2270
|
-
try {
|
|
2271
|
-
let prefillResult;
|
|
2272
|
-
const prefillStart = performance.now();
|
|
2273
|
-
try {
|
|
2274
|
-
prefillResult = await this._prefillToHidden(inputIds, {
|
|
2275
|
-
...opts,
|
|
2276
|
-
_embeddingOnly: options.__skipStateSnapshot === true,
|
|
2277
|
-
});
|
|
2278
|
-
} catch (error) {
|
|
2279
|
-
if (shouldRetryWithFinitenessFallback(error)) {
|
|
2280
|
-
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillWithEmbedding. Retrying with F32 precision.`);
|
|
2281
|
-
prefillResult = await this._retryWithPersistentFinitenessFallback(
|
|
2282
|
-
opts,
|
|
2283
|
-
'prefillWithEmbedding',
|
|
2284
|
-
1,
|
|
2285
|
-
() => this._prefillToHidden(inputIds, {
|
|
2286
|
-
...opts,
|
|
2287
|
-
_embeddingOnly: options.__skipStateSnapshot === true,
|
|
2288
|
-
}),
|
|
2289
|
-
prefillStartSeqLen
|
|
2290
|
-
);
|
|
2291
|
-
} else {
|
|
2292
|
-
throw error;
|
|
2293
|
-
}
|
|
2294
|
-
}
|
|
2295
|
-
const prefillMs = performance.now() - prefillStart;
|
|
2296
|
-
|
|
2297
|
-
const {
|
|
2298
|
-
numTokens,
|
|
2299
|
-
config,
|
|
2300
|
-
startPos,
|
|
2301
|
-
activationDtype,
|
|
2302
|
-
activationBytes,
|
|
2303
|
-
currentRecorder,
|
|
2304
|
-
recordProfile,
|
|
2305
|
-
currentHiddenBuffer,
|
|
2306
|
-
} = prefillResult;
|
|
2307
|
-
|
|
2308
|
-
// Ensure prefill work completes before readback.
|
|
2309
|
-
let submitWaitMs = 0;
|
|
2310
|
-
const submitWaitStart = performance.now();
|
|
2311
|
-
if (currentRecorder) {
|
|
2312
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
2313
|
-
await currentRecorder.submitAndWait();
|
|
2314
|
-
await recordProfile(currentRecorder);
|
|
2315
|
-
} else {
|
|
2316
|
-
const device = getDevice();
|
|
2317
|
-
if (device) {
|
|
2318
|
-
await device.queue.onSubmittedWorkDone();
|
|
2319
|
-
}
|
|
2320
|
-
}
|
|
2321
|
-
submitWaitMs = performance.now() - submitWaitStart;
|
|
2322
|
-
|
|
2323
|
-
if (!allowReadback('pipeline.prefill.embedding')) {
|
|
2324
|
-
throw new Error('GPU readback disabled; cannot return embedding');
|
|
2325
|
-
}
|
|
2326
|
-
|
|
2327
|
-
let embedding;
|
|
2328
|
-
let tokenEmbeddings = null;
|
|
2329
|
-
let logits = null;
|
|
2330
|
-
let hiddenBytes = 0;
|
|
2331
|
-
let readbackMs = 0;
|
|
2332
|
-
let decodeHiddenMs = 0;
|
|
2333
|
-
let finalNormMs = 0;
|
|
2334
|
-
let extractMs = 0;
|
|
2335
|
-
let logitsMs = 0;
|
|
2336
|
-
try {
|
|
2337
|
-
if (options.__returnSequenceLogits === true) {
|
|
2338
|
-
const logitsStart = performance.now();
|
|
2339
|
-
logits = await computeLogits(
|
|
2340
|
-
currentHiddenBuffer,
|
|
2341
|
-
numTokens,
|
|
2342
|
-
getLogitsWeights(this.#state),
|
|
2343
|
-
getLogitsConfig(this.#state),
|
|
2344
|
-
this.#state.useGPU,
|
|
2345
|
-
this.#state.debugFlags,
|
|
2346
|
-
undefined,
|
|
2347
|
-
undefined,
|
|
2348
|
-
this.#state.runtimeConfig.shared.debug.probes,
|
|
2349
|
-
{ lastPositionOnly: false },
|
|
2350
|
-
this.#state.operatorDiagnostics
|
|
2351
|
-
);
|
|
2352
|
-
logitsMs = performance.now() - logitsStart;
|
|
2353
|
-
}
|
|
2354
|
-
const hiddenSize = config.hiddenSize;
|
|
2355
|
-
hiddenBytes = numTokens * hiddenSize * activationBytes;
|
|
2356
|
-
const readbackStart = performance.now();
|
|
2357
|
-
const hiddenData = await readBuffer(currentHiddenBuffer, hiddenBytes);
|
|
2358
|
-
readbackMs = performance.now() - readbackStart;
|
|
2359
|
-
if (hiddenData.byteLength === 0) {
|
|
2360
|
-
throw new Error('GPU readback disabled; cannot return embedding');
|
|
2361
|
-
}
|
|
2362
|
-
const decodeStart = performance.now();
|
|
2363
|
-
const hiddenStates = decodeReadback(hiddenData, activationDtype);
|
|
2364
|
-
decodeHiddenMs = performance.now() - decodeStart;
|
|
2365
|
-
const finalNormStart = performance.now();
|
|
2366
|
-
const finalNormWeights = await this._getFinalNormWeights();
|
|
2367
|
-
finalNormMs = performance.now() - finalNormStart;
|
|
2368
|
-
const extractStart = performance.now();
|
|
2369
|
-
if (options.__returnTokenEmbeddings === true) {
|
|
2370
|
-
tokenEmbeddings = this._extractTokenEmbeddingsFromHidden(
|
|
2371
|
-
hiddenStates,
|
|
2372
|
-
numTokens,
|
|
2373
|
-
hiddenSize,
|
|
2374
|
-
finalNormWeights,
|
|
2375
|
-
config
|
|
2376
|
-
);
|
|
2377
|
-
}
|
|
2378
|
-
embedding = extractEmbeddingFromHidden(
|
|
2379
|
-
hiddenStates,
|
|
2380
|
-
numTokens,
|
|
2381
|
-
hiddenSize,
|
|
2382
|
-
opts.embeddingMode,
|
|
2383
|
-
finalNormWeights,
|
|
2384
|
-
config,
|
|
2385
|
-
this.#state.embeddingPostprocessor,
|
|
2386
|
-
tokenEmbeddings,
|
|
2387
|
-
this.#state.weights.get('final_norm_bias') ?? null
|
|
2388
|
-
);
|
|
2389
|
-
extractMs = performance.now() - extractStart;
|
|
2390
|
-
} finally {
|
|
2391
|
-
releaseBuffer(currentHiddenBuffer);
|
|
2392
|
-
}
|
|
2393
|
-
|
|
2394
|
-
this.#state.currentSeqLen = startPos + numTokens;
|
|
2395
|
-
const phase = {
|
|
2396
|
-
totalMs: performance.now() - totalStart,
|
|
2397
|
-
inputMs,
|
|
2398
|
-
prefillMs,
|
|
2399
|
-
submitWaitMs,
|
|
2400
|
-
readbackMs,
|
|
2401
|
-
decodeHiddenMs,
|
|
2402
|
-
finalNormMs,
|
|
2403
|
-
extractMs,
|
|
2404
|
-
logitsMs,
|
|
2405
|
-
hiddenBytes,
|
|
2406
|
-
tokens: numTokens,
|
|
2407
|
-
activationDtype,
|
|
2408
|
-
prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
|
|
2409
|
-
prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
|
|
2410
|
-
prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
|
|
2411
|
-
prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
|
|
2412
|
-
prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
|
|
2413
|
-
gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
|
|
2414
|
-
};
|
|
2415
|
-
|
|
2416
|
-
// Batch embedding skips expensive KV cache clone and linear attention clone
|
|
2417
|
-
// since the caller will reset immediately after extracting the embedding.
|
|
2418
|
-
if (options.__skipStateSnapshot) {
|
|
2419
|
-
return {
|
|
2420
|
-
cache: null,
|
|
2421
|
-
seqLen: this.#state.currentSeqLen,
|
|
2422
|
-
tokens: inputIds,
|
|
2423
|
-
embedding,
|
|
2424
|
-
tokenEmbeddings,
|
|
2425
|
-
logits,
|
|
2426
|
-
embeddingMode: opts.embeddingMode,
|
|
2427
|
-
phase,
|
|
2428
|
-
linearAttention: null,
|
|
2429
|
-
};
|
|
2430
|
-
}
|
|
2431
|
-
|
|
2432
|
-
const snapshot = this.#state.kvCache?.clone();
|
|
2433
|
-
if (!snapshot) {
|
|
2434
|
-
throw new Error('KV cache unavailable after prefill');
|
|
2435
|
-
}
|
|
2436
|
-
|
|
2437
|
-
return {
|
|
2438
|
-
cache: snapshot,
|
|
2439
|
-
seqLen: this.#state.currentSeqLen,
|
|
2440
|
-
tokens: inputIds,
|
|
2441
|
-
embedding,
|
|
2442
|
-
tokenEmbeddings,
|
|
2443
|
-
logits,
|
|
2444
|
-
embeddingMode: opts.embeddingMode,
|
|
2445
|
-
phase,
|
|
2446
|
-
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
2447
|
-
};
|
|
2448
|
-
} finally {
|
|
2449
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
2450
|
-
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
2451
|
-
? {
|
|
2452
|
-
enabled: true,
|
|
2453
|
-
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
2454
|
-
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
2455
|
-
}
|
|
2456
|
-
: null;
|
|
2457
|
-
this.#state.operatorDiagnostics = null;
|
|
2458
|
-
}
|
|
549
|
+
return prefillWithEmbeddingImpl.apply(this, arguments);
|
|
2459
550
|
}
|
|
2460
551
|
|
|
2461
552
|
async prefillWithLogits(prompt, options = {}) {
|
|
2462
|
-
|
|
2463
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2464
|
-
throw new Error('Generation already in progress');
|
|
2465
|
-
}
|
|
2466
|
-
assertIncrementalDecodeSupport(this.#state, 'prefillWithLogits');
|
|
2467
|
-
const totalStart = performance.now();
|
|
2468
|
-
this._resetDecodeRuntimeState();
|
|
2469
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2470
|
-
this.#state.stats.prefillProfileSteps = [];
|
|
2471
|
-
const opts = resolvePrefillOptions(this.#state, options);
|
|
2472
|
-
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
2473
|
-
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
2474
|
-
if (diagnosticsEnabled) {
|
|
2475
|
-
const captureConfig = {
|
|
2476
|
-
...createDefaultCaptureConfig(),
|
|
2477
|
-
enabled: true,
|
|
2478
|
-
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
2479
|
-
...(options?.diagnostics?.captureConfig ?? {}),
|
|
2480
|
-
};
|
|
2481
|
-
validateCaptureConfig(captureConfig);
|
|
2482
|
-
this.#state.operatorDiagnostics = {
|
|
2483
|
-
enabled: true,
|
|
2484
|
-
captureConfig,
|
|
2485
|
-
emitter: new OperatorEventEmitter({
|
|
2486
|
-
modelHash: this.#state.manifest?.modelId ?? null,
|
|
2487
|
-
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
2488
|
-
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
2489
|
-
}),
|
|
2490
|
-
tsirFixture: null,
|
|
2491
|
-
};
|
|
2492
|
-
}
|
|
2493
|
-
try {
|
|
2494
|
-
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
|
|
2495
|
-
|
|
2496
|
-
const snapshot = this.#state.kvCache?.clone();
|
|
2497
|
-
if (!snapshot) {
|
|
2498
|
-
throw new Error('KV cache unavailable after prefill');
|
|
2499
|
-
}
|
|
2500
|
-
|
|
2501
|
-
return {
|
|
2502
|
-
cache: snapshot,
|
|
2503
|
-
seqLen: this.#state.currentSeqLen,
|
|
2504
|
-
tokens: inputIds,
|
|
2505
|
-
logits,
|
|
2506
|
-
phase: {
|
|
2507
|
-
...(prefillPhase ?? {}),
|
|
2508
|
-
totalMs: performance.now() - totalStart,
|
|
2509
|
-
},
|
|
2510
|
-
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
2511
|
-
};
|
|
2512
|
-
} finally {
|
|
2513
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
2514
|
-
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
2515
|
-
? {
|
|
2516
|
-
enabled: true,
|
|
2517
|
-
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
2518
|
-
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
2519
|
-
}
|
|
2520
|
-
: null;
|
|
2521
|
-
this.#state.operatorDiagnostics = null;
|
|
2522
|
-
}
|
|
553
|
+
return prefillWithLogitsImpl.apply(this, arguments);
|
|
2523
554
|
}
|
|
2524
555
|
|
|
2525
556
|
async prefillWithTokenLogits(prompt, tokenIds, options = {}) {
|
|
2526
|
-
|
|
2527
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2528
|
-
throw new Error('Generation already in progress');
|
|
2529
|
-
}
|
|
2530
|
-
assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogits');
|
|
2531
|
-
const totalStart = performance.now();
|
|
2532
|
-
const selectedTokenIds = normalizeSelectedLogitTokenIds(
|
|
2533
|
-
tokenIds,
|
|
2534
|
-
this.#state.modelConfig.vocabSize,
|
|
2535
|
-
'prefillWithTokenLogits.tokenIds'
|
|
2536
|
-
);
|
|
2537
|
-
this._resetDecodeRuntimeState();
|
|
2538
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2539
|
-
this.#state.stats.prefillProfileSteps = [];
|
|
2540
|
-
const opts = resolvePrefillOptions(this.#state, options);
|
|
2541
|
-
opts._selectedLogitTokenIds = selectedTokenIds;
|
|
2542
|
-
try {
|
|
2543
|
-
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogits');
|
|
2544
|
-
const logitsByTokenId = {};
|
|
2545
|
-
for (let index = 0; index < selectedTokenIds.length; index += 1) {
|
|
2546
|
-
logitsByTokenId[selectedTokenIds[index]] = logits[index];
|
|
2547
|
-
}
|
|
2548
|
-
return {
|
|
2549
|
-
seqLen: this.#state.currentSeqLen,
|
|
2550
|
-
tokens: inputIds,
|
|
2551
|
-
tokenIds: selectedTokenIds,
|
|
2552
|
-
logits,
|
|
2553
|
-
logitsByTokenId,
|
|
2554
|
-
phase: {
|
|
2555
|
-
...(prefillPhase ?? {}),
|
|
2556
|
-
totalMs: performance.now() - totalStart,
|
|
2557
|
-
selectedTokenCount: selectedTokenIds.length,
|
|
2558
|
-
},
|
|
2559
|
-
};
|
|
2560
|
-
} finally {
|
|
2561
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
2562
|
-
}
|
|
557
|
+
return prefillWithTokenLogitsImpl.apply(this, arguments);
|
|
2563
558
|
}
|
|
2564
559
|
|
|
2565
560
|
async prefillWithTokenLogitsFromKV(prefix, prompt, tokenIds, options = {}) {
|
|
2566
|
-
|
|
2567
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2568
|
-
throw new Error('Generation already in progress');
|
|
2569
|
-
}
|
|
2570
|
-
assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogitsFromKV');
|
|
2571
|
-
const totalStart = performance.now();
|
|
2572
|
-
if (!prefix || typeof prefix !== 'object' || !prefix.cache) {
|
|
2573
|
-
throw new Error('prefillWithTokenLogitsFromKV.prefix must be a KV cache snapshot.');
|
|
2574
|
-
}
|
|
2575
|
-
if (!Number.isInteger(prefix.seqLen) || prefix.seqLen < 0) {
|
|
2576
|
-
throw new Error('prefillWithTokenLogitsFromKV.prefix.seqLen must be a non-negative integer.');
|
|
2577
|
-
}
|
|
2578
|
-
if (!Array.isArray(prefix.tokens)) {
|
|
2579
|
-
throw new Error('prefillWithTokenLogitsFromKV.prefix.tokens must be an array.');
|
|
2580
|
-
}
|
|
2581
|
-
const selectedTokenIds = normalizeSelectedLogitTokenIds(
|
|
2582
|
-
tokenIds,
|
|
2583
|
-
this.#state.modelConfig.vocabSize,
|
|
2584
|
-
'prefillWithTokenLogitsFromKV.tokenIds'
|
|
2585
|
-
);
|
|
2586
|
-
|
|
2587
|
-
this.#state.kvCache = prefix.cache.clone();
|
|
2588
|
-
if (this.#state.useGPU && this.#state.kvCache) {
|
|
2589
|
-
const device = getDevice();
|
|
2590
|
-
if (device) {
|
|
2591
|
-
this.#state.kvCache.setGPUContext({ device });
|
|
2592
|
-
}
|
|
2593
|
-
}
|
|
2594
|
-
if (
|
|
2595
|
-
hasLinearAttentionLayers(this.#state.modelConfig.layerTypes)
|
|
2596
|
-
&& prefix.linearAttention == null
|
|
2597
|
-
) {
|
|
2598
|
-
throw new Error(
|
|
2599
|
-
'Prefix snapshot is missing linear_attention recurrent state. ' +
|
|
2600
|
-
'Regenerate the prefix snapshot using the current runtime.'
|
|
2601
|
-
);
|
|
2602
|
-
}
|
|
2603
|
-
this.#state.linearAttentionRuntime = restoreLinearAttentionRuntime(
|
|
2604
|
-
this.#state.linearAttentionRuntime,
|
|
2605
|
-
prefix.linearAttention ?? null
|
|
2606
|
-
);
|
|
2607
|
-
this.#state.currentSeqLen = prefix.seqLen;
|
|
2608
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2609
|
-
this.#state.stats.prefillProfileSteps = [];
|
|
2610
|
-
this.#state.decodeRing?.reset();
|
|
2611
|
-
|
|
2612
|
-
const opts = resolvePrefillOptions(this.#state, options);
|
|
2613
|
-
opts._selectedLogitTokenIds = selectedTokenIds;
|
|
2614
|
-
try {
|
|
2615
|
-
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogitsFromKV');
|
|
2616
|
-
const logitsByTokenId = {};
|
|
2617
|
-
for (let index = 0; index < selectedTokenIds.length; index += 1) {
|
|
2618
|
-
logitsByTokenId[selectedTokenIds[index]] = logits[index];
|
|
2619
|
-
}
|
|
2620
|
-
return {
|
|
2621
|
-
seqLen: this.#state.currentSeqLen,
|
|
2622
|
-
prefixTokens: prefix.tokens,
|
|
2623
|
-
tokens: inputIds,
|
|
2624
|
-
tokenIds: selectedTokenIds,
|
|
2625
|
-
logits,
|
|
2626
|
-
logitsByTokenId,
|
|
2627
|
-
phase: {
|
|
2628
|
-
...(prefillPhase ?? {}),
|
|
2629
|
-
totalMs: performance.now() - totalStart,
|
|
2630
|
-
selectedTokenCount: selectedTokenIds.length,
|
|
2631
|
-
prefixTokens: prefix.tokens.length,
|
|
2632
|
-
},
|
|
2633
|
-
};
|
|
2634
|
-
} finally {
|
|
2635
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
2636
|
-
}
|
|
561
|
+
return prefillWithTokenLogitsFromKVImpl.apply(this, arguments);
|
|
2637
562
|
}
|
|
2638
563
|
|
|
2639
|
-
|
|
2640
564
|
async *generateWithPrefixKV(prefix, prompt, options = {}) {
|
|
2641
|
-
|
|
2642
|
-
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
2643
|
-
assertIncrementalDecodeSupport(this.#state, 'generateWithPrefixKV');
|
|
2644
|
-
|
|
2645
|
-
validateCallTimeOptions(options);
|
|
2646
|
-
|
|
2647
|
-
// Apply snapshot
|
|
2648
|
-
this.#state.kvCache = prefix.cache.clone();
|
|
2649
|
-
if (this.#state.useGPU && this.#state.kvCache) {
|
|
2650
|
-
const device = getDevice();
|
|
2651
|
-
if (device) {
|
|
2652
|
-
this.#state.kvCache.setGPUContext({ device });
|
|
2653
|
-
}
|
|
2654
|
-
}
|
|
2655
|
-
if (
|
|
2656
|
-
hasLinearAttentionLayers(this.#state.modelConfig.layerTypes)
|
|
2657
|
-
&& prefix.linearAttention == null
|
|
2658
|
-
) {
|
|
2659
|
-
throw new Error(
|
|
2660
|
-
'Prefix snapshot is missing linear_attention recurrent state. ' +
|
|
2661
|
-
'Regenerate the prefix snapshot using the current runtime.'
|
|
2662
|
-
);
|
|
2663
|
-
}
|
|
2664
|
-
this.#state.linearAttentionRuntime = restoreLinearAttentionRuntime(
|
|
2665
|
-
this.#state.linearAttentionRuntime,
|
|
2666
|
-
prefix.linearAttention ?? null
|
|
2667
|
-
);
|
|
2668
|
-
this.#state.currentSeqLen = prefix.seqLen;
|
|
2669
|
-
|
|
2670
|
-
this.#state.isGenerating = true;
|
|
2671
|
-
this.#state.decodeStepCount = 0;
|
|
2672
|
-
resetActiveExecutionPlan(this.#state);
|
|
2673
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2674
|
-
this.#state.stats.gpuTimeDecodeMs = undefined;
|
|
2675
|
-
this.#state.stats.prefillProfileSteps = [];
|
|
2676
|
-
this.#state.decodeRing?.reset();
|
|
2677
|
-
this.#state.stats.decodeRecordMs = 0;
|
|
2678
|
-
this.#state.stats.decodeRecordOps = 0;
|
|
2679
|
-
this.#state.stats.decodeRecordPasses = 0;
|
|
2680
|
-
this.#state.stats.decodeRecordOpLabels = {};
|
|
2681
|
-
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
2682
|
-
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
2683
|
-
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
2684
|
-
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
2685
|
-
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
2686
|
-
this.#state.stats.prefillRecordMs = 0;
|
|
2687
|
-
this.#state.stats.prefillRecordOps = 0;
|
|
2688
|
-
this.#state.stats.prefillRecordPasses = 0;
|
|
2689
|
-
this.#state.stats.prefillRecordOpLabels = {};
|
|
2690
|
-
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
2691
|
-
this.#state.stats.ttftMs = 0;
|
|
2692
|
-
const startTime = performance.now();
|
|
2693
|
-
|
|
2694
|
-
const opts = resolveGenerateOptions(this.#state, options);
|
|
2695
|
-
|
|
2696
|
-
try {
|
|
2697
|
-
const processedPrompt = resolvePromptInput(this.#state, prompt, opts.useChatTemplate, 'generateWithPrefixKV');
|
|
2698
|
-
|
|
2699
|
-
const inputIds = this.#state.tokenizer.encode(processedPrompt);
|
|
2700
|
-
this._assertTokenIdsInRange(inputIds, 'generateWithPrefixKV.encode');
|
|
2701
|
-
const generatedIds = [...prefix.tokens, ...inputIds];
|
|
2702
|
-
const promptTokenCount = generatedIds.length;
|
|
2703
|
-
this.#state.stats.prefillTokens = inputIds.length;
|
|
2704
|
-
|
|
2705
|
-
const prefillStart = performance.now();
|
|
2706
|
-
const prefillLogits = await this._prefill(inputIds, opts);
|
|
2707
|
-
this.#state.stats.prefillTimeMs = performance.now() - prefillStart;
|
|
2708
|
-
|
|
2709
|
-
applyRepetitionPenalty(prefillLogits, generatedIds, opts.repetitionPenalty);
|
|
2710
|
-
const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
|
|
2711
|
-
const firstToken = sample(prefillLogits, {
|
|
2712
|
-
temperature: opts.temperature,
|
|
2713
|
-
topP: opts.topP,
|
|
2714
|
-
topK: opts.topK,
|
|
2715
|
-
padTokenId,
|
|
2716
|
-
seed: opts.seed,
|
|
2717
|
-
suppressTokenIds: opts.suppressTokenIds,
|
|
2718
|
-
});
|
|
2719
|
-
|
|
2720
|
-
const decodeRuntime = {
|
|
2721
|
-
stopTokenIds: this.#state.modelConfig.stopTokenIds,
|
|
2722
|
-
eosToken: this.#state.tokenizer.getSpecialTokens?.()?.eos,
|
|
2723
|
-
stopSequenceStart: promptTokenCount,
|
|
2724
|
-
decodeToken: (tokenId) => this.#state.tokenizer.decode([tokenId], true, false),
|
|
2725
|
-
logBatchPath: false,
|
|
2726
|
-
};
|
|
2727
|
-
generatedIds.push(firstToken);
|
|
2728
|
-
this.#state.stats.ttftMs = performance.now() - startTime;
|
|
2729
|
-
const firstText = resolveTokenText(
|
|
2730
|
-
this.#state.tokenizer,
|
|
2731
|
-
[firstToken],
|
|
2732
|
-
`[${firstToken}]`,
|
|
2733
|
-
(tokens) => this.#state.tokenizer?.decode?.(tokens, true, false),
|
|
2734
|
-
(tokens) => this.#state.tokenizer?.decode?.(tokens, false, false)
|
|
2735
|
-
);
|
|
2736
|
-
yield firstText;
|
|
2737
|
-
if (options.onToken) options.onToken(firstToken, firstText);
|
|
2738
|
-
|
|
2739
|
-
if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
|
|
2740
|
-
this._recordStopReason('stop-token-or-sequence', firstToken);
|
|
2741
|
-
this.#state.stats.decodeTimeMs = 0;
|
|
2742
|
-
this.#state.stats.tokensGenerated = 1;
|
|
2743
|
-
this.#state.stats.decodeTokens = 1;
|
|
2744
|
-
} else {
|
|
2745
|
-
yield* this._runDecodeLoop(generatedIds, opts, options, decodeRuntime);
|
|
2746
|
-
}
|
|
2747
|
-
this.#state.stats.totalTimeMs = performance.now() - startTime;
|
|
2748
|
-
} finally {
|
|
2749
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
2750
|
-
resetActiveExecutionPlan(this.#state);
|
|
2751
|
-
this.#state.isGenerating = false;
|
|
2752
|
-
}
|
|
565
|
+
return yield* generateWithPrefixKVImpl.apply(this, arguments);
|
|
2753
566
|
}
|
|
2754
567
|
|
|
2755
568
|
// ==========================================================================
|
|
@@ -2757,1226 +570,34 @@ export class PipelineGenerator {
|
|
|
2757
570
|
// ==========================================================================
|
|
2758
571
|
|
|
2759
572
|
async *_runDecodeLoop(generatedIds, opts, options, runtime) {
|
|
2760
|
-
|
|
2761
|
-
stopTokenIds,
|
|
2762
|
-
eosToken,
|
|
2763
|
-
stopSequenceStart,
|
|
2764
|
-
decodeToken,
|
|
2765
|
-
logBatchPath = false,
|
|
2766
|
-
emitMode = 'text',
|
|
2767
|
-
} = runtime;
|
|
2768
|
-
|
|
2769
|
-
let tokensGenerated = 1;
|
|
2770
|
-
markKernelCacheWarmed();
|
|
2771
|
-
|
|
2772
|
-
// Step 4: Lazily initialise PLE buffer cache for decode-path slice reuse.
|
|
2773
|
-
const pleHiddenSize = Number(this.#state.modelConfig.hiddenSizePerLayerInput ?? 0);
|
|
2774
|
-
if (pleHiddenSize > 0 && !this.#state.pleCache) {
|
|
2775
|
-
const activationDtype = resolveActiveExecutionPlan(this.#state).activationDtype;
|
|
2776
|
-
const bpe = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
|
|
2777
|
-
this.#state.pleCache = createPleBufferCache(
|
|
2778
|
-
this.#state.modelConfig.numLayers,
|
|
2779
|
-
pleHiddenSize * bpe,
|
|
2780
|
-
);
|
|
2781
|
-
}
|
|
2782
|
-
if (pleHiddenSize > 0) {
|
|
2783
|
-
await primePleDecodeRuntimeCache(this.#state, generatedIds);
|
|
2784
|
-
}
|
|
2785
|
-
|
|
2786
|
-
const decodeStart = performance.now();
|
|
2787
|
-
const resolvedPerLayerInputsSession = resolvePerLayerInputsSession(
|
|
2788
|
-
this.#state.modelConfig.perLayerInputsSession ?? null,
|
|
2789
|
-
this.#state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
2790
|
-
);
|
|
2791
|
-
const lmHead = this.#state.weights.get('lm_head');
|
|
2792
|
-
const embedBuffer = this.#state.weights.get('embed');
|
|
2793
|
-
const hasCpuWeights = isCpuWeightBuffer(lmHead)
|
|
2794
|
-
|| isCpuWeightBuffer(embedBuffer)
|
|
2795
|
-
|| lmHead instanceof Float32Array
|
|
2796
|
-
|| embedBuffer instanceof Float32Array;
|
|
2797
|
-
const hasLinearLayers = hasLinearAttentionLayers(this.#state.modelConfig.layerTypes);
|
|
2798
|
-
const replayPrefillDecode = usesReplayPrefillDecode(this.#state);
|
|
2799
|
-
const gpuSamplingAvailable = isGPUSamplingAvailable() && !hasCpuWeights;
|
|
2800
|
-
const hasSuppressedSamplingTokens = Array.isArray(opts.suppressTokenIds) && opts.suppressTokenIds.length > 0;
|
|
2801
|
-
const hasRangeBackedPerLayerInputs = hasRangeBackedPerLayerInputEmbeddings({
|
|
2802
|
-
config: this.#state.modelConfig,
|
|
2803
|
-
weights: this.#state.weights,
|
|
2804
|
-
});
|
|
2805
|
-
const hasGpuSplitPerLayerInputs = hasGpuSplitPerLayerInputEmbeddings({
|
|
2806
|
-
config: this.#state.modelConfig,
|
|
2807
|
-
weights: this.#state.weights,
|
|
2808
|
-
perLayerInputsSession: resolvedPerLayerInputsSession,
|
|
2809
|
-
});
|
|
2810
|
-
const pleHotVocabularyRuntime = getPleHotVocabularyRuntime({ weights: this.#state.weights });
|
|
2811
|
-
const resolveCurrentHotVocabularyBatchDecodeAvailable = () => resolveHotVocabularyBatchDecodeAvailability({
|
|
2812
|
-
hasRangeBackedPerLayerInputs,
|
|
2813
|
-
pleHotVocabularyRuntime,
|
|
2814
|
-
tokenId: generatedIds[generatedIds.length - 1] ?? null,
|
|
2815
|
-
});
|
|
2816
|
-
const initialHotVocabularyBatchDecodeAvailable = resolveCurrentHotVocabularyBatchDecodeAvailable();
|
|
2817
|
-
const executionPlan = opts.executionPlan;
|
|
2818
|
-
let useBatchPath = replayPrefillDecode
|
|
2819
|
-
? false
|
|
2820
|
-
: shouldUseBatchDecode({
|
|
2821
|
-
batchSize: executionPlan.batchSize,
|
|
2822
|
-
useGPU: this.#state.useGPU,
|
|
2823
|
-
gpuSamplingAvailable,
|
|
2824
|
-
disableMultiTokenDecode: executionPlan.disableMultiTokenDecode || hasSuppressedSamplingTokens,
|
|
2825
|
-
disableCommandBatching: executionPlan.disableCommandBatching,
|
|
2826
|
-
isBdpaPagedLayout: this.#state.kvCache?.layout === 'bdpa_paged',
|
|
2827
|
-
finitenessFallbackWindowOpen: this._hasFinitenessFallbackWindow(),
|
|
2828
|
-
hasLinearAttentionLayers: hasLinearLayers,
|
|
2829
|
-
selfSpeculationEnabled: opts.speculation?.mode === 'self' && !initialHotVocabularyBatchDecodeAvailable,
|
|
2830
|
-
hasRangeBackedPerLayerInputs,
|
|
2831
|
-
});
|
|
2832
|
-
if (!useBatchPath) {
|
|
2833
|
-
let reason = null;
|
|
2834
|
-
if (replayPrefillDecode) reason = 'replay_prefill_decode';
|
|
2835
|
-
else if (hasCpuWeights) reason = 'cpu_weights';
|
|
2836
|
-
else if (!this.#state.useGPU) reason = 'no_gpu';
|
|
2837
|
-
else if (!gpuSamplingAvailable) reason = 'no_gpu_sampling';
|
|
2838
|
-
else if (executionPlan.disableCommandBatching) reason = 'command_batching_disabled';
|
|
2839
|
-
else if (hasSuppressedSamplingTokens) reason = 'sampling_suppression_requires_cpu_logits';
|
|
2840
|
-
else if (executionPlan.disableMultiTokenDecode) reason = 'multi_token_decode_disabled';
|
|
2841
|
-
else if (executionPlan.batchSize <= 1) reason = 'batch_size_1';
|
|
2842
|
-
else if (this.#state.kvCache?.layout === 'bdpa_paged') reason = 'bdpa_paged_layout';
|
|
2843
|
-
else if (this._hasFinitenessFallbackWindow()) reason = 'finiteness_fallback_window';
|
|
2844
|
-
else if (hasLinearLayers) reason = 'linear_attention_layers';
|
|
2845
|
-
this.#state.stats.decodeMode = replayPrefillDecode
|
|
2846
|
-
? 'replay_prefill'
|
|
2847
|
-
: (opts.speculation?.mode === 'self' ? 'self_speculation' : 'single_token');
|
|
2848
|
-
this.#state.stats.batchGuardReason = reason;
|
|
2849
|
-
} else {
|
|
2850
|
-
this.#state.stats.decodeMode = hasRangeBackedPerLayerInputs
|
|
2851
|
-
? 'batched_gpu_stepwise_ple'
|
|
2852
|
-
: 'batched_gpu';
|
|
2853
|
-
this.#state.stats.batchGuardReason = null;
|
|
2854
|
-
}
|
|
2855
|
-
|
|
2856
|
-
const readbackInterval = executionPlan.readbackInterval;
|
|
2857
|
-
const intervalBatches = readbackInterval == null ? 1 : readbackInterval;
|
|
2858
|
-
const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
|
|
2859
|
-
|
|
2860
|
-
const decodeSingleTokenViaLogits = async () => this._decodeNextTokenViaLogits(generatedIds, opts);
|
|
2861
|
-
|
|
2862
|
-
if (logBatchPath && useBatchPath) {
|
|
2863
|
-
log.debug(
|
|
2864
|
-
'Pipeline',
|
|
2865
|
-
`Using batch decode path with batchSize=${executionPlan.batchSize}, stopCheckMode=${executionPlan.stopCheckMode}, readbackInterval=${readbackInterval}`
|
|
2866
|
-
);
|
|
2867
|
-
}
|
|
2868
|
-
|
|
2869
|
-
while (tokensGenerated < opts.maxTokens) {
|
|
2870
|
-
if (options.signal?.aborted) {
|
|
2871
|
-
this._recordStopReason('aborted');
|
|
2872
|
-
break;
|
|
2873
|
-
}
|
|
2874
|
-
if (this._hasFinitenessFallbackWindow() && useBatchPath) {
|
|
2875
|
-
useBatchPath = false;
|
|
2876
|
-
}
|
|
2877
|
-
const hotVocabularyBatchDecodeAvailable = resolveCurrentHotVocabularyBatchDecodeAvailable();
|
|
2878
|
-
|
|
2879
|
-
if (useBatchPath) {
|
|
2880
|
-
const remaining = opts.maxTokens - tokensGenerated;
|
|
2881
|
-
const maxBatchDecodeTokens = resolveMaxBatchDecodeTokens({
|
|
2882
|
-
hasHotVocabularyBatchDecode: hotVocabularyBatchDecodeAvailable,
|
|
2883
|
-
hasGpuSplitPerLayerInputs,
|
|
2884
|
-
hasLinearAttentionLayers: hasLinearLayers,
|
|
2885
|
-
modelId: this.#state.modelConfig.modelId ?? executionPlan.kernelPathId,
|
|
2886
|
-
activationDtype: executionPlan.activationDtype,
|
|
2887
|
-
currentSeqLen: this.#state.currentSeqLen,
|
|
2888
|
-
maxDecodeTokens: opts.maxTokens,
|
|
2889
|
-
numLayers: this.#state.modelConfig.numLayers,
|
|
2890
|
-
hiddenSize: this.#state.modelConfig.hiddenSize,
|
|
2891
|
-
configuredMaxBatchDecodeTokens: executionPlan.maxBatchDecodeTokens,
|
|
2892
|
-
});
|
|
2893
|
-
const requestedBatchTokens = executionPlan.batchSize * intervalBatches;
|
|
2894
|
-
const boundedBatchTokens = maxBatchDecodeTokens == null
|
|
2895
|
-
? requestedBatchTokens
|
|
2896
|
-
: Math.min(requestedBatchTokens, maxBatchDecodeTokens);
|
|
2897
|
-
const thisBatchSize = Math.min(boundedBatchTokens, remaining);
|
|
2898
|
-
this.#state.batchingStats.requestedBatchTokens = Math.max(
|
|
2899
|
-
this.#state.batchingStats.requestedBatchTokens ?? 0,
|
|
2900
|
-
requestedBatchTokens
|
|
2901
|
-
);
|
|
2902
|
-
this.#state.batchingStats.effectiveBatchTokens = Math.max(
|
|
2903
|
-
this.#state.batchingStats.effectiveBatchTokens ?? 0,
|
|
2904
|
-
thisBatchSize
|
|
2905
|
-
);
|
|
2906
|
-
if (maxBatchDecodeTokens != null) {
|
|
2907
|
-
this.#state.batchingStats.maxBatchTokenCap = Math.max(
|
|
2908
|
-
this.#state.batchingStats.maxBatchTokenCap ?? 0,
|
|
2909
|
-
maxBatchDecodeTokens
|
|
2910
|
-
);
|
|
2911
|
-
}
|
|
2912
|
-
if (thisBatchSize < requestedBatchTokens) {
|
|
2913
|
-
this.#state.batchingStats.batchClampCount = (this.#state.batchingStats.batchClampCount ?? 0) + 1;
|
|
2914
|
-
}
|
|
2915
|
-
const lastToken = generatedIds[generatedIds.length - 1];
|
|
2916
|
-
const boundedExecutionPlan = thisBatchSize < requestedBatchTokens
|
|
2917
|
-
? {
|
|
2918
|
-
...executionPlan,
|
|
2919
|
-
batchSize: Math.min(executionPlan.batchSize, thisBatchSize),
|
|
2920
|
-
readbackInterval: readbackInterval == null
|
|
2921
|
-
? null
|
|
2922
|
-
: Math.min(intervalBatches, thisBatchSize),
|
|
2923
|
-
}
|
|
2924
|
-
: executionPlan;
|
|
2925
|
-
const batchOpts = boundedExecutionPlan === executionPlan
|
|
2926
|
-
? opts
|
|
2927
|
-
: { ...opts, executionPlan: boundedExecutionPlan };
|
|
2928
|
-
|
|
2929
|
-
try {
|
|
2930
|
-
const batchResult = await this._generateNTokensGPU(lastToken, thisBatchSize, generatedIds, batchOpts);
|
|
2931
|
-
let batchTokens = [];
|
|
2932
|
-
let hitStop = false;
|
|
2933
|
-
let stopTokenId = null;
|
|
2934
|
-
for (const tokenId of batchResult.tokens) {
|
|
2935
|
-
if (isStopToken(tokenId, stopTokenIds, eosToken)) {
|
|
2936
|
-
hitStop = true;
|
|
2937
|
-
stopTokenId = tokenId;
|
|
2938
|
-
break;
|
|
2939
|
-
}
|
|
2940
|
-
generatedIds.push(tokenId);
|
|
2941
|
-
tokensGenerated++;
|
|
2942
|
-
if (emitMode === 'token') {
|
|
2943
|
-
yield tokenId;
|
|
2944
|
-
if (options.onToken) options.onToken(tokenId, '');
|
|
2945
|
-
batchTokens.push({ id: tokenId, text: '' });
|
|
2946
|
-
} else {
|
|
2947
|
-
const tokenText = decodeToken(tokenId);
|
|
2948
|
-
yield tokenText;
|
|
2949
|
-
if (options.onToken) options.onToken(tokenId, tokenText);
|
|
2950
|
-
batchTokens.push({ id: tokenId, text: tokenText });
|
|
2951
|
-
}
|
|
2952
|
-
if (batchTokens.length === executionPlan.batchSize) {
|
|
2953
|
-
if (options.onBatch) options.onBatch(batchTokens);
|
|
2954
|
-
batchTokens = [];
|
|
2955
|
-
}
|
|
2956
|
-
}
|
|
2957
|
-
if (batchTokens.length > 0 && options.onBatch) options.onBatch(batchTokens);
|
|
2958
|
-
if (opts.stopSequences.length > 0) {
|
|
2959
|
-
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
2960
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
2961
|
-
this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
|
|
2962
|
-
break;
|
|
2963
|
-
}
|
|
2964
|
-
}
|
|
2965
|
-
if (hitStop) {
|
|
2966
|
-
this._recordStopReason('stop-token', stopTokenId);
|
|
2967
|
-
break;
|
|
2968
|
-
}
|
|
2969
|
-
if (shouldDisableBatchDecodeAfterShortBatch({
|
|
2970
|
-
hitStop,
|
|
2971
|
-
actualCount: batchResult.actualCount,
|
|
2972
|
-
requestedCount: thisBatchSize,
|
|
2973
|
-
})) {
|
|
2974
|
-
useBatchPath = false;
|
|
2975
|
-
continue;
|
|
2976
|
-
}
|
|
2977
|
-
} catch (error) {
|
|
2978
|
-
log.warn('Pipeline', `Batch decode failed, falling back to single-token: ${error}`);
|
|
2979
|
-
useBatchPath = false;
|
|
2980
|
-
let nextToken;
|
|
2981
|
-
try {
|
|
2982
|
-
nextToken = await decodeSingleTokenViaLogits();
|
|
2983
|
-
} catch (singleTokenError) {
|
|
2984
|
-
if (this._shouldUseFinitenessFallback(singleTokenError, `decode-batch-step-${tokensGenerated}`)) {
|
|
2985
|
-
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at batch step ${tokensGenerated}. Truncating KV cache and retrying token with F32 precision.`);
|
|
2986
|
-
nextToken = await this._retryDecodeStepWithFinitenessWindow(
|
|
2987
|
-
generatedIds,
|
|
2988
|
-
opts,
|
|
2989
|
-
`decode-batch-step-${tokensGenerated}`
|
|
2990
|
-
);
|
|
2991
|
-
} else {
|
|
2992
|
-
throw singleTokenError;
|
|
2993
|
-
}
|
|
2994
|
-
}
|
|
2995
|
-
generatedIds.push(nextToken);
|
|
2996
|
-
tokensGenerated++;
|
|
2997
|
-
if (emitMode === 'token') {
|
|
2998
|
-
yield nextToken;
|
|
2999
|
-
if (options.onToken) options.onToken(nextToken, '');
|
|
3000
|
-
} else {
|
|
3001
|
-
const tokenText = decodeToken(nextToken);
|
|
3002
|
-
yield tokenText;
|
|
3003
|
-
if (options.onToken) options.onToken(nextToken, tokenText);
|
|
3004
|
-
}
|
|
3005
|
-
this._consumeFinitenessFallbackToken(opts);
|
|
3006
|
-
if (isStopToken(nextToken, stopTokenIds, eosToken)) {
|
|
3007
|
-
this._recordStopReason('stop-token', nextToken);
|
|
3008
|
-
break;
|
|
3009
|
-
}
|
|
3010
|
-
}
|
|
3011
|
-
} else if (opts.speculation?.mode === 'self') {
|
|
3012
|
-
// Self-speculation: decode one base token plus a configurable burst of
|
|
3013
|
-
// speculative tokens per iteration. Same-model speculation always
|
|
3014
|
-
// accepts under greedy because the model is deterministic — both base
|
|
3015
|
-
// and speculative use the same weights and state. The benefit is
|
|
3016
|
-
// amortizing per-iteration overhead for models where batch decode is
|
|
3017
|
-
// disabled (e.g., linear attention).
|
|
3018
|
-
const speculativeBurstTokens = opts.speculation.tokens;
|
|
3019
|
-
if (!Number.isInteger(speculativeBurstTokens) || speculativeBurstTokens < 1) {
|
|
3020
|
-
throw new Error('[Pipeline] resolved self-speculation tokens must be a positive integer.');
|
|
3021
|
-
}
|
|
3022
|
-
const doSpecDecode = hasLinearLayers
|
|
3023
|
-
? () => this._decodeStep(generatedIds, opts)
|
|
3024
|
-
: decodeSingleTokenViaLogits;
|
|
3025
|
-
|
|
3026
|
-
// Base decode
|
|
3027
|
-
let baseToken;
|
|
3028
|
-
try {
|
|
3029
|
-
baseToken = await doSpecDecode();
|
|
3030
|
-
} catch (error) {
|
|
3031
|
-
if (this._shouldUseFinitenessFallback(error, `spec-base-${tokensGenerated}`)) {
|
|
3032
|
-
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated} (speculation:base). Retrying.`);
|
|
3033
|
-
baseToken = await this._retryDecodeStepWithFinitenessWindow(generatedIds, opts, `spec-base-${tokensGenerated}`);
|
|
3034
|
-
} else {
|
|
3035
|
-
throw error;
|
|
3036
|
-
}
|
|
3037
|
-
}
|
|
3038
|
-
generatedIds.push(baseToken);
|
|
3039
|
-
tokensGenerated++;
|
|
3040
|
-
if (emitMode === 'token') {
|
|
3041
|
-
yield baseToken;
|
|
3042
|
-
if (options.onToken) options.onToken(baseToken, '');
|
|
3043
|
-
} else {
|
|
3044
|
-
const text = decodeToken(baseToken);
|
|
3045
|
-
yield text;
|
|
3046
|
-
if (options.onToken) options.onToken(baseToken, text);
|
|
3047
|
-
}
|
|
3048
|
-
this._consumeFinitenessFallbackToken(opts);
|
|
3049
|
-
|
|
3050
|
-
if (isStopToken(baseToken, stopTokenIds, eosToken)) {
|
|
3051
|
-
this._recordStopReason('stop-token', baseToken);
|
|
3052
|
-
break;
|
|
3053
|
-
}
|
|
3054
|
-
if (tokensGenerated >= opts.maxTokens) break;
|
|
3055
|
-
if (opts.stopSequences.length > 0) {
|
|
3056
|
-
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
3057
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
3058
|
-
this._recordStopReason('stop-sequence', baseToken);
|
|
3059
|
-
break;
|
|
3060
|
-
}
|
|
3061
|
-
}
|
|
3062
|
-
|
|
3063
|
-
for (let specIndex = 0; specIndex < speculativeBurstTokens; specIndex += 1) {
|
|
3064
|
-
if (tokensGenerated >= opts.maxTokens) {
|
|
3065
|
-
break;
|
|
3066
|
-
}
|
|
3067
|
-
let specToken;
|
|
3068
|
-
try {
|
|
3069
|
-
specToken = await doSpecDecode();
|
|
3070
|
-
} catch (error) {
|
|
3071
|
-
if (this._shouldUseFinitenessFallback(error, `spec-extra-${tokensGenerated}`)) {
|
|
3072
|
-
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated} (speculation:spec). Retrying.`);
|
|
3073
|
-
specToken = await this._retryDecodeStepWithFinitenessWindow(generatedIds, opts, `spec-extra-${tokensGenerated}`);
|
|
3074
|
-
} else {
|
|
3075
|
-
throw error;
|
|
3076
|
-
}
|
|
3077
|
-
}
|
|
3078
|
-
generatedIds.push(specToken);
|
|
3079
|
-
tokensGenerated++;
|
|
3080
|
-
this.#state.stats.speculationAttempts = (this.#state.stats.speculationAttempts ?? 0) + 1;
|
|
3081
|
-
this.#state.stats.speculationAccepted = (this.#state.stats.speculationAccepted ?? 0) + 1;
|
|
3082
|
-
if (emitMode === 'token') {
|
|
3083
|
-
yield specToken;
|
|
3084
|
-
if (options.onToken) options.onToken(specToken, '');
|
|
3085
|
-
} else {
|
|
3086
|
-
const text = decodeToken(specToken);
|
|
3087
|
-
yield text;
|
|
3088
|
-
if (options.onToken) options.onToken(specToken, text);
|
|
3089
|
-
}
|
|
3090
|
-
this._consumeFinitenessFallbackToken(opts);
|
|
3091
|
-
|
|
3092
|
-
if (opts.debug || opts.benchmark) {
|
|
3093
|
-
const elapsedMs = performance.now() - decodeStart;
|
|
3094
|
-
const tokPerSec = (tokensGenerated / elapsedMs) * 1000;
|
|
3095
|
-
log.debug('Decode', `#${tokensGenerated} speculation:self (${tokPerSec.toFixed(2)} tok/s avg)`);
|
|
3096
|
-
}
|
|
3097
|
-
|
|
3098
|
-
if (isStopToken(specToken, stopTokenIds, eosToken)) {
|
|
3099
|
-
this._recordStopReason('stop-token', specToken);
|
|
3100
|
-
break;
|
|
3101
|
-
}
|
|
3102
|
-
if (opts.stopSequences.length > 0) {
|
|
3103
|
-
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
3104
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
3105
|
-
this._recordStopReason('stop-sequence', specToken);
|
|
3106
|
-
break;
|
|
3107
|
-
}
|
|
3108
|
-
}
|
|
3109
|
-
}
|
|
3110
|
-
if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken)) {
|
|
3111
|
-
this._recordStopReason('stop-token', generatedIds[generatedIds.length - 1]);
|
|
3112
|
-
break;
|
|
3113
|
-
}
|
|
3114
|
-
if (opts.stopSequences.length > 0) {
|
|
3115
|
-
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
3116
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
3117
|
-
this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
|
|
3118
|
-
break;
|
|
3119
|
-
}
|
|
3120
|
-
}
|
|
3121
|
-
} else {
|
|
3122
|
-
const tokenStart = performance.now();
|
|
3123
|
-
let nextToken;
|
|
3124
|
-
try {
|
|
3125
|
-
nextToken = hasLinearLayers
|
|
3126
|
-
? await this._decodeStep(generatedIds, opts)
|
|
3127
|
-
: await decodeSingleTokenViaLogits();
|
|
3128
|
-
} catch (error) {
|
|
3129
|
-
if (this._shouldUseFinitenessFallback(error, `decode-step-${tokensGenerated}`)) {
|
|
3130
|
-
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated}. Truncating KV cache and retrying token with F32 precision.`);
|
|
3131
|
-
nextToken = await this._retryDecodeStepWithFinitenessWindow(
|
|
3132
|
-
generatedIds,
|
|
3133
|
-
opts,
|
|
3134
|
-
`decode-step-${tokensGenerated}`
|
|
3135
|
-
);
|
|
3136
|
-
} else {
|
|
3137
|
-
throw error;
|
|
3138
|
-
}
|
|
3139
|
-
}
|
|
3140
|
-
const tokenTime = performance.now() - tokenStart;
|
|
3141
|
-
generatedIds.push(nextToken);
|
|
3142
|
-
tokensGenerated++;
|
|
3143
|
-
|
|
3144
|
-
// Step 5: Fire-and-forget prefetch of next token's PLE row.
|
|
3145
|
-
if (pleHiddenSize > 0) {
|
|
3146
|
-
const pleWeights = this.#state.weights.get('per_layer_inputs');
|
|
3147
|
-
if (pleWeights?.embedTokensPerLayer) {
|
|
3148
|
-
this.#state.plePrefetchPending = prefetchPerLayerRow(
|
|
3149
|
-
nextToken,
|
|
3150
|
-
pleWeights.embedTokensPerLayer,
|
|
3151
|
-
this.#state.modelConfig.numLayers * pleHiddenSize,
|
|
3152
|
-
resolvedPerLayerInputsSession,
|
|
3153
|
-
);
|
|
3154
|
-
}
|
|
3155
|
-
}
|
|
3156
|
-
|
|
3157
|
-
const tokenText = emitMode === 'token' ? '' : decodeToken(nextToken);
|
|
3158
|
-
if (emitMode === 'token') {
|
|
3159
|
-
yield nextToken;
|
|
3160
|
-
if (options.onToken) options.onToken(nextToken, '');
|
|
3161
|
-
} else {
|
|
3162
|
-
yield tokenText;
|
|
3163
|
-
if (options.onToken) options.onToken(nextToken, tokenText);
|
|
3164
|
-
}
|
|
3165
|
-
this._consumeFinitenessFallbackToken(opts);
|
|
3166
|
-
|
|
3167
|
-
if (opts.debug || opts.benchmark) {
|
|
3168
|
-
const elapsedMs = performance.now() - decodeStart;
|
|
3169
|
-
const tokPerSec = (tokensGenerated / elapsedMs) * 1000;
|
|
3170
|
-
log.debug('Decode', `#${tokensGenerated} "${tokenText}" ${tokenTime.toFixed(0)}ms (${tokPerSec.toFixed(2)} tok/s avg)`);
|
|
3171
|
-
}
|
|
3172
|
-
|
|
3173
|
-
if (isStopToken(nextToken, stopTokenIds, eosToken)) {
|
|
3174
|
-
this._recordStopReason('stop-token', nextToken);
|
|
3175
|
-
break;
|
|
3176
|
-
}
|
|
3177
|
-
if (opts.stopSequences.length > 0) {
|
|
3178
|
-
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
3179
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
3180
|
-
this._recordStopReason('stop-sequence', nextToken);
|
|
3181
|
-
break;
|
|
3182
|
-
}
|
|
3183
|
-
}
|
|
3184
|
-
}
|
|
3185
|
-
}
|
|
3186
|
-
|
|
3187
|
-
if (!this.#state.stats.stopReason) {
|
|
3188
|
-
this._recordStopReason(tokensGenerated >= opts.maxTokens ? 'max-tokens' : 'completed');
|
|
3189
|
-
}
|
|
3190
|
-
this.#state.stats.decodeTimeMs = performance.now() - decodeStart;
|
|
3191
|
-
this.#state.stats.tokensGenerated = tokensGenerated;
|
|
3192
|
-
this.#state.stats.decodeTokens = tokensGenerated;
|
|
3193
|
-
this.#state.stats.batching = { ...this.#state.batchingStats };
|
|
573
|
+
return yield* _runDecodeLoopImpl.apply(this, arguments);
|
|
3194
574
|
}
|
|
3195
575
|
|
|
3196
576
|
async _prefillToHidden(inputIds, opts) {
|
|
3197
|
-
|
|
3198
|
-
return this._prefill(inputIds, { ...opts, _returnHidden: true });
|
|
577
|
+
return _prefillToHiddenImpl.apply(this, arguments);
|
|
3199
578
|
}
|
|
3200
579
|
|
|
3201
|
-
|
|
3202
580
|
async _prefill(inputIds, opts) {
|
|
3203
|
-
|
|
3204
|
-
const config = this.#state.modelConfig;
|
|
3205
|
-
const selectedLogitTokenIds = normalizeSelectedLogitTokenIds(
|
|
3206
|
-
opts?._selectedLogitTokenIds,
|
|
3207
|
-
config.vocabSize,
|
|
3208
|
-
'_prefill.selectedLogitTokenIds'
|
|
3209
|
-
);
|
|
3210
|
-
const startPos = this.#state.currentSeqLen;
|
|
3211
|
-
const tracePrefillEnabled = isTraceEnabled('perf');
|
|
3212
|
-
const prefillTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
3213
|
-
const returnHidden = opts?._returnHidden === true;
|
|
3214
|
-
const embeddingInputIds = resolvePrefillEmbeddingInputIds(
|
|
3215
|
-
inputIds,
|
|
3216
|
-
opts?.embeddingInputSpan ?? null,
|
|
3217
|
-
'_prefill'
|
|
3218
|
-
);
|
|
3219
|
-
const multimodalBidirectionalSpan = resolvePrefillMultimodalBidirectionalSpan(
|
|
3220
|
-
inputIds,
|
|
3221
|
-
opts?.multimodalBidirectionalSpan ?? null,
|
|
3222
|
-
'_prefill'
|
|
3223
|
-
);
|
|
3224
|
-
if (embeddingInputIds !== inputIds) {
|
|
3225
|
-
this._assertTokenIdsInRange(embeddingInputIds, '_prefill.embeddingInputIds');
|
|
3226
|
-
}
|
|
3227
|
-
const embeddingOverride = normalizePrefixEmbeddingOverride(
|
|
3228
|
-
opts?.embeddingOverrides ?? null,
|
|
3229
|
-
config.hiddenSize,
|
|
3230
|
-
numTokens,
|
|
3231
|
-
'_prefill'
|
|
3232
|
-
);
|
|
3233
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
3234
|
-
this.#state.stats.prefillProfileSteps = [];
|
|
3235
|
-
|
|
3236
|
-
if (startPos === 0 && hasLinearAttentionLayers(config.layerTypes)) {
|
|
3237
|
-
this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
|
|
3238
|
-
}
|
|
3239
|
-
if (startPos === 0) {
|
|
3240
|
-
for (const [, convState] of this.#state.convLayerStates) {
|
|
3241
|
-
if (convState.convStateGPU && convState.hiddenSize && convState.kernelSize) {
|
|
3242
|
-
uploadData(convState.convStateGPU, new Float32Array(convState.hiddenSize * (convState.kernelSize - 1)));
|
|
3243
|
-
}
|
|
3244
|
-
}
|
|
3245
|
-
}
|
|
3246
|
-
|
|
3247
|
-
const embedBufferRaw = this.#state.weights.get('embed');
|
|
3248
|
-
if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
|
|
3249
|
-
throw new Error('Embed buffer not found or not a supported buffer type');
|
|
3250
|
-
}
|
|
3251
|
-
const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
|
|
3252
|
-
const embedDtype = isCpuWeightBuffer(embedBufferRaw)
|
|
3253
|
-
? embedBufferRaw.dtype
|
|
3254
|
-
: getWeightDtype(embedBufferRaw);
|
|
3255
|
-
const embedMetadata = getWeightMetadata(embedBufferRaw);
|
|
3256
|
-
if (opts.debug) {
|
|
3257
|
-
const embedSize = isGpuBufferInstance(embedBuffer) ? embedBuffer.size : 'N/A';
|
|
3258
|
-
log.debug('Pipeline', `Embed buffer: type=${embedBuffer?.constructor?.name}, size=${embedSize}, dtype=${embedDtype}`);
|
|
3259
|
-
}
|
|
3260
|
-
|
|
3261
|
-
const device = getDevice();
|
|
3262
|
-
const useCheckpoints = opts.debugLayers && opts.debugLayers.length > 0;
|
|
3263
|
-
const disableCommandBatching = shouldDisablePrefillCommandBatching(
|
|
3264
|
-
this.#state,
|
|
3265
|
-
opts,
|
|
3266
|
-
multimodalBidirectionalSpan
|
|
3267
|
-
);
|
|
3268
|
-
const createRecorder = (label) => {
|
|
3269
|
-
if (!device || disableCommandBatching) return undefined;
|
|
3270
|
-
return opts.profile
|
|
3271
|
-
? createProfilingRecorder(label, device)
|
|
3272
|
-
: createCommandRecorder(label, { recordLabels: opts.debug === true || opts.benchmark === true }, device);
|
|
3273
|
-
};
|
|
3274
|
-
const recorder = createRecorder('prefill');
|
|
3275
|
-
const debugCheckBuffer = this.#state.debug
|
|
3276
|
-
? (buffer, label, numTokens, expectedDim) =>
|
|
3277
|
-
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
3278
|
-
: undefined;
|
|
3279
|
-
const context = buildLayerContext(
|
|
3280
|
-
this.#state,
|
|
3281
|
-
recorder,
|
|
3282
|
-
false,
|
|
3283
|
-
opts.debugLayers,
|
|
3284
|
-
debugCheckBuffer,
|
|
3285
|
-
opts.executionPlan
|
|
3286
|
-
);
|
|
3287
|
-
context.trainingCapture = opts?._trainingCapture ?? null;
|
|
3288
|
-
context.currentTokenIds = inputIds;
|
|
3289
|
-
context.diffusionGemmaDecoder = opts?._diffusionGemmaDecoder === true;
|
|
3290
|
-
context.skipKVCacheWrites = returnHidden
|
|
3291
|
-
&& opts?._embeddingOnly === true
|
|
3292
|
-
&& this.#state.runtimeConfig?.inference?.session?.skipEmbeddingKVCacheWrites === true;
|
|
3293
|
-
context.multimodalBidirectionalSpan = multimodalBidirectionalSpan == null
|
|
3294
|
-
? null
|
|
3295
|
-
: {
|
|
3296
|
-
start: startPos + multimodalBidirectionalSpan.offset,
|
|
3297
|
-
length: multimodalBidirectionalSpan.length,
|
|
3298
|
-
};
|
|
3299
|
-
let gpuTimePrefillMs = 0;
|
|
3300
|
-
let hasGpuTimePrefill = false;
|
|
3301
|
-
const recordProfile = async (rec) => {
|
|
3302
|
-
if (!opts.profile || !rec) return;
|
|
3303
|
-
const timings = rec.isProfilingEnabled()
|
|
3304
|
-
? await rec.resolveProfileTimings()
|
|
3305
|
-
: null;
|
|
3306
|
-
const total = sumProfileTimings(timings);
|
|
3307
|
-
if (total !== null) {
|
|
3308
|
-
gpuTimePrefillMs += total;
|
|
3309
|
-
hasGpuTimePrefill = true;
|
|
3310
|
-
}
|
|
3311
|
-
if (timings || opts.executionObserver) {
|
|
3312
|
-
recordPrefillProfileStep(this.#state, {
|
|
3313
|
-
label: rec.label,
|
|
3314
|
-
timings,
|
|
3315
|
-
totalMs: total ?? undefined,
|
|
3316
|
-
recorderStats: rec.getStats(),
|
|
3317
|
-
});
|
|
3318
|
-
if (!opts.executionObserver) {
|
|
3319
|
-
log.warn('Profile', `Prefill (${rec.label}):`);
|
|
3320
|
-
log.warn('Profile', CommandRecorder.formatProfileReport(timings));
|
|
3321
|
-
}
|
|
3322
|
-
}
|
|
3323
|
-
};
|
|
3324
|
-
|
|
3325
|
-
const benchmarkSubmits = opts.debug;
|
|
3326
|
-
if (benchmarkSubmits) {
|
|
3327
|
-
setTrackSubmits(true);
|
|
3328
|
-
resetSubmitStats();
|
|
3329
|
-
}
|
|
3330
|
-
|
|
3331
|
-
const preserveBufferAcrossRecorderSubmit = (buffer, activeRecorder, label) => {
|
|
3332
|
-
if (!activeRecorder || !isGpuBufferInstance(buffer)) {
|
|
3333
|
-
return buffer;
|
|
3334
|
-
}
|
|
3335
|
-
const carryBuffer = acquireBuffer(
|
|
3336
|
-
buffer.size,
|
|
3337
|
-
typeof buffer.usage === 'number' ? buffer.usage : undefined,
|
|
3338
|
-
label
|
|
3339
|
-
);
|
|
3340
|
-
activeRecorder.getEncoder().copyBufferToBuffer(buffer, 0, carryBuffer, 0, buffer.size);
|
|
3341
|
-
activeRecorder.trackTemporaryBuffer(buffer);
|
|
3342
|
-
return carryBuffer;
|
|
3343
|
-
};
|
|
3344
|
-
|
|
3345
|
-
const activationDtype = opts.executionPlan?.activationDtype ?? this._getEffectiveActivationDtype();
|
|
3346
|
-
const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
|
|
3347
|
-
if (tracePrefillEnabled) {
|
|
3348
|
-
trace.perf('Prefill phase start', {
|
|
3349
|
-
numTokens,
|
|
3350
|
-
startPos,
|
|
3351
|
-
numLayers: config.numLayers,
|
|
3352
|
-
activationDtype,
|
|
3353
|
-
returnHidden,
|
|
3354
|
-
});
|
|
3355
|
-
}
|
|
3356
|
-
const embedTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
3357
|
-
let baseEmbeddings = await embed(embeddingInputIds, embedBuffer, {
|
|
3358
|
-
hiddenSize: config.hiddenSize,
|
|
3359
|
-
vocabSize: config.vocabSize,
|
|
3360
|
-
scaleEmbeddings: config.scaleEmbeddings,
|
|
3361
|
-
embeddingScale: config.embeddingScale,
|
|
3362
|
-
debug: opts.debug,
|
|
3363
|
-
recorder,
|
|
3364
|
-
transpose: this.#state.embeddingTranspose,
|
|
3365
|
-
debugProbes: this.#state.runtimeConfig.shared.debug.probes,
|
|
3366
|
-
operatorDiagnostics: this.#state.operatorDiagnostics,
|
|
3367
|
-
activationDtype,
|
|
3368
|
-
embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
|
|
3369
|
-
embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
|
|
3370
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
3371
|
-
});
|
|
3372
|
-
if (tracePrefillEnabled) {
|
|
3373
|
-
trace.perf('Prefill embed complete', {
|
|
3374
|
-
numTokens,
|
|
3375
|
-
elapsedMs: performance.now() - embedTraceStart,
|
|
3376
|
-
});
|
|
3377
|
-
}
|
|
3378
|
-
let hiddenStates = baseEmbeddings;
|
|
3379
|
-
let perLayerInputs = null;
|
|
3380
|
-
const perLayerInputsTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
3381
|
-
try {
|
|
3382
|
-
hiddenStates = await applyPrefixEmbeddingOverride(
|
|
3383
|
-
baseEmbeddings,
|
|
3384
|
-
embeddingOverride,
|
|
3385
|
-
config.hiddenSize,
|
|
3386
|
-
'_prefill',
|
|
3387
|
-
{
|
|
3388
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
3389
|
-
transitionDeclaredBy: resolvePrefixEmbeddingOverrideTransitionDeclaredBy(this.#state.executionV1State),
|
|
3390
|
-
}
|
|
3391
|
-
);
|
|
3392
|
-
perLayerInputs = await preparePerLayerInputs(
|
|
3393
|
-
embeddingInputIds,
|
|
3394
|
-
embeddingInputIds === inputIds ? hiddenStates : baseEmbeddings,
|
|
3395
|
-
context,
|
|
3396
|
-
{
|
|
3397
|
-
numTokens,
|
|
3398
|
-
pleCache: this.#state.pleCache ?? null,
|
|
3399
|
-
}
|
|
3400
|
-
);
|
|
3401
|
-
if (tracePrefillEnabled) {
|
|
3402
|
-
trace.perf('Prefill per-layer inputs complete', {
|
|
3403
|
-
numTokens,
|
|
3404
|
-
elapsedMs: performance.now() - perLayerInputsTraceStart,
|
|
3405
|
-
materialized: Array.isArray(perLayerInputs),
|
|
3406
|
-
});
|
|
3407
|
-
}
|
|
3408
|
-
} catch (error) {
|
|
3409
|
-
if (isGpuBufferInstance(hiddenStates?.buffer)) {
|
|
3410
|
-
releaseBuffer(hiddenStates.buffer);
|
|
3411
|
-
}
|
|
3412
|
-
if (hiddenStates === baseEmbeddings) {
|
|
3413
|
-
baseEmbeddings = null;
|
|
3414
|
-
}
|
|
3415
|
-
hiddenStates = null;
|
|
3416
|
-
throw error;
|
|
3417
|
-
} finally {
|
|
3418
|
-
if (hiddenStates !== baseEmbeddings && isGpuBufferInstance(baseEmbeddings?.buffer)) {
|
|
3419
|
-
releaseBuffer(baseEmbeddings.buffer);
|
|
3420
|
-
}
|
|
3421
|
-
baseEmbeddings = null;
|
|
3422
|
-
}
|
|
3423
|
-
|
|
3424
|
-
if (opts.debug && isGpuBufferInstance(hiddenStates)) {
|
|
3425
|
-
if (recorder) {
|
|
3426
|
-
hiddenStates = createTensor(
|
|
3427
|
-
preserveBufferAcrossRecorderSubmit(hiddenStates.buffer, recorder, 'prefill_embed_carry'),
|
|
3428
|
-
hiddenStates.dtype,
|
|
3429
|
-
hiddenStates.shape,
|
|
3430
|
-
hiddenStates.label
|
|
3431
|
-
);
|
|
3432
|
-
recordPrefillRecorderStats(this.#state, recorder);
|
|
3433
|
-
await recorder.submitAndWait();
|
|
3434
|
-
await recordProfile(recorder);
|
|
3435
|
-
}
|
|
3436
|
-
const debugReadbackSize = this.#state.runtimeConfig.shared.debug.pipeline.readbackSampleSize;
|
|
3437
|
-
const sample = await readBuffer(hiddenStates, Math.min(debugReadbackSize, hiddenStates.size));
|
|
3438
|
-
const f32 = decodeReadback(sample, activationDtype);
|
|
3439
|
-
const nanCount = f32.filter(x => !Number.isFinite(x)).length;
|
|
3440
|
-
let maxAbs = 0;
|
|
3441
|
-
for (let i = 0; i < f32.length; i++) {
|
|
3442
|
-
const abs = Math.abs(f32[i]);
|
|
3443
|
-
if (abs > maxAbs) maxAbs = abs;
|
|
3444
|
-
}
|
|
3445
|
-
const first8 = Array.from(f32).slice(0, 8).map(x => x.toFixed(4)).join(', ');
|
|
3446
|
-
log.debug('Pipeline', `After embed: buffer.label=${hiddenStates.label}, buffer.size=${hiddenStates.size}, maxAbs=${maxAbs.toFixed(4)}`);
|
|
3447
|
-
log.debug('Pipeline', `After embed first8=[${first8}], nan=${nanCount}/${f32.length}`);
|
|
3448
|
-
}
|
|
3449
|
-
|
|
3450
|
-
if (opts.debug) {
|
|
3451
|
-
log.debug('Pipeline', `LAYER_LOOP_START: numLayers=${config.numLayers}, useGPU=${context.useGPU}`);
|
|
3452
|
-
}
|
|
3453
|
-
|
|
3454
|
-
if (this.#state.finitenessBuffer) {
|
|
3455
|
-
const device = getDevice();
|
|
3456
|
-
if (device) {
|
|
3457
|
-
device.queue.writeBuffer(this.#state.finitenessBuffer, 0, FINITENESS_RESET_WORDS);
|
|
3458
|
-
}
|
|
3459
|
-
}
|
|
3460
|
-
|
|
3461
|
-
let currentRecorder = recorder;
|
|
3462
|
-
|
|
3463
|
-
// Chunked recorder submission: submit every N layers to release tracked intermediate
|
|
3464
|
-
// buffers, preventing unbounded memory growth during large prefills. Critical for
|
|
3465
|
-
// replay_prefill models where each decode step re-runs a prefill-style layer pass.
|
|
3466
|
-
const prefillRecorderChunkLayers = resolvePrefillRecorderChunkLayers({
|
|
3467
|
-
configuredPrefillChunkLayers: this.#state.runtimeConfig?.inference?.session?.prefillChunkLayers,
|
|
3468
|
-
hasGpuSplitPerLayerInputs: context.perLayerInputsSession?.materialization === 'gpu_split_tables',
|
|
3469
|
-
numTokens,
|
|
3470
|
-
});
|
|
3471
|
-
|
|
3472
|
-
let currentHiddenBuffer = hiddenStates.buffer;
|
|
3473
|
-
let prefillRecordMs = 0;
|
|
3474
|
-
let prefillSubmitWaitMs = 0;
|
|
3475
|
-
const layerLoopTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
3476
|
-
const traceLayerHealthEnabled = isTraceEnabled('logits');
|
|
3477
|
-
try {
|
|
3478
|
-
for (let l = 0; l < config.numLayers; l++) {
|
|
3479
|
-
// Per-layer hard cancellation: when the caller's AbortSignal aborts,
|
|
3480
|
-
// exit the prefill loop between layer dispatches rather than
|
|
3481
|
-
// continuing to burn GPU on superseded work. Granularity is one
|
|
3482
|
-
// layer (~5-50ms), which is the fastest cancel granularity WebGPU
|
|
3483
|
-
// exposes today.
|
|
3484
|
-
if (opts?.signal?.aborted) {
|
|
3485
|
-
const reason = typeof opts.signal.reason === "string" ? opts.signal.reason : "Doppler: prefill aborted";
|
|
3486
|
-
const err = new Error(reason);
|
|
3487
|
-
err.name = "AbortError";
|
|
3488
|
-
err.code = "ABORT_ERR";
|
|
3489
|
-
throw err;
|
|
3490
|
-
}
|
|
3491
|
-
context.recorder = currentRecorder;
|
|
3492
|
-
context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
|
|
3493
|
-
|
|
3494
|
-
const prevBuffer = currentHiddenBuffer;
|
|
3495
|
-
const layerRecordStart = performance.now();
|
|
3496
|
-
const layerOutput = await processLayer(l, currentHiddenBuffer, numTokens, true, context);
|
|
3497
|
-
prefillRecordMs += performance.now() - layerRecordStart;
|
|
3498
|
-
if (!isGpuBufferInstance(layerOutput)) throw new Error('Expected GPUBuffer from processLayer');
|
|
3499
|
-
currentHiddenBuffer = layerOutput;
|
|
3500
|
-
releasePerLayerInputBuffer(
|
|
3501
|
-
context.perLayerInputBuffer,
|
|
3502
|
-
currentRecorder,
|
|
3503
|
-
context.decodeBuffers,
|
|
3504
|
-
this.#state.pleCache ?? null
|
|
3505
|
-
);
|
|
3506
|
-
if (perLayerInputs) {
|
|
3507
|
-
perLayerInputs[l] = null;
|
|
3508
|
-
}
|
|
3509
|
-
context.perLayerInputBuffer = null;
|
|
3510
|
-
|
|
3511
|
-
const isCheckpoint = useCheckpoints && opts.debugLayers?.includes(l);
|
|
3512
|
-
const isChunkBoundary = !isCheckpoint
|
|
3513
|
-
&& !traceLayerHealthEnabled
|
|
3514
|
-
&& currentRecorder
|
|
3515
|
-
&& l < config.numLayers - 1
|
|
3516
|
-
&& (l + 1) % prefillRecorderChunkLayers === 0;
|
|
3517
|
-
|
|
3518
|
-
if (isCheckpoint && currentRecorder) {
|
|
3519
|
-
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3520
|
-
currentHiddenBuffer,
|
|
3521
|
-
currentRecorder,
|
|
3522
|
-
'prefill_checkpoint_carry'
|
|
3523
|
-
);
|
|
3524
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3525
|
-
await currentRecorder.submitAndWait();
|
|
3526
|
-
await recordProfile(currentRecorder);
|
|
3527
|
-
currentRecorder = undefined;
|
|
3528
|
-
}
|
|
3529
|
-
|
|
3530
|
-
if (traceLayerHealthEnabled && currentRecorder) {
|
|
3531
|
-
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3532
|
-
currentHiddenBuffer,
|
|
3533
|
-
currentRecorder,
|
|
3534
|
-
'prefill_trace_layer_health_carry'
|
|
3535
|
-
);
|
|
3536
|
-
const traceSubmitStart = performance.now();
|
|
3537
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3538
|
-
await currentRecorder.submitAndWait();
|
|
3539
|
-
await recordProfile(currentRecorder);
|
|
3540
|
-
prefillSubmitWaitMs += performance.now() - traceSubmitStart;
|
|
3541
|
-
await traceActivationHealth(
|
|
3542
|
-
`PREFILL_LAYER_${l}_HEALTH`,
|
|
3543
|
-
currentHiddenBuffer,
|
|
3544
|
-
activationDtype,
|
|
3545
|
-
numTokens * config.hiddenSize
|
|
3546
|
-
);
|
|
3547
|
-
currentRecorder = l < config.numLayers - 1
|
|
3548
|
-
? createRecorder('prefill-trace')
|
|
3549
|
-
: undefined;
|
|
3550
|
-
}
|
|
3551
|
-
|
|
3552
|
-
const shouldDebug = opts.debug && currentHiddenBuffer && (!recorder || isCheckpoint);
|
|
3553
|
-
if (shouldDebug && !currentRecorder) {
|
|
3554
|
-
const device = getDevice();
|
|
3555
|
-
if (device) {
|
|
3556
|
-
if (allowReadback(`pipeline.prefill.layer-${l}`)) {
|
|
3557
|
-
try {
|
|
3558
|
-
const sampleSize = config.hiddenSize * activationBytes;
|
|
3559
|
-
const lastTokenOffset = (numTokens - 1) * config.hiddenSize * activationBytes;
|
|
3560
|
-
const readback = await readBufferSlice(currentHiddenBuffer, lastTokenOffset, sampleSize);
|
|
3561
|
-
const data = decodeReadback(readback, activationDtype);
|
|
3562
|
-
let min = Infinity;
|
|
3563
|
-
let max = -Infinity;
|
|
3564
|
-
let maxAbs = 0;
|
|
3565
|
-
for (const v of data) {
|
|
3566
|
-
if (!Number.isFinite(v)) continue;
|
|
3567
|
-
if (v < min) min = v;
|
|
3568
|
-
if (v > max) max = v;
|
|
3569
|
-
const av = Math.abs(v);
|
|
3570
|
-
if (av > maxAbs) maxAbs = av;
|
|
3571
|
-
}
|
|
3572
|
-
const sample = Array.from(data).slice(0, 3).map(x => x.toFixed(3)).join(', ');
|
|
3573
|
-
log.debug('Pipeline', `LAYER_${l}_LAST[pos=${numTokens - 1}]: min=${min.toFixed(3)}, max=${max.toFixed(3)}, maxAbs=${maxAbs.toFixed(2)}, sample=[${sample}]`);
|
|
3574
|
-
} catch (e) {
|
|
3575
|
-
log.debug('Pipeline', `LAYER_${l}_LAST: error reading buffer: ${e}`);
|
|
3576
|
-
}
|
|
3577
|
-
}
|
|
3578
|
-
}
|
|
3579
|
-
}
|
|
3580
|
-
|
|
3581
|
-
if (isCheckpoint && useCheckpoints && l < config.numLayers - 1) {
|
|
3582
|
-
currentRecorder = createRecorder('prefill-cont');
|
|
3583
|
-
}
|
|
3584
|
-
|
|
3585
|
-
if (prevBuffer !== currentHiddenBuffer) {
|
|
3586
|
-
if (currentRecorder) {
|
|
3587
|
-
currentRecorder.trackTemporaryBuffer(prevBuffer);
|
|
3588
|
-
} else {
|
|
3589
|
-
releaseBuffer(prevBuffer);
|
|
3590
|
-
}
|
|
3591
|
-
}
|
|
3592
|
-
|
|
3593
|
-
if (isChunkBoundary) {
|
|
3594
|
-
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3595
|
-
currentHiddenBuffer,
|
|
3596
|
-
currentRecorder,
|
|
3597
|
-
'prefill_chunk_carry'
|
|
3598
|
-
);
|
|
3599
|
-
// Chunk boundary exists only to bound intermediate buffer lifetime.
|
|
3600
|
-
// When the runtime opts into async chunk submit AND profile timings
|
|
3601
|
-
// are not being collected, skip the CPU-GPU wait: queue order is
|
|
3602
|
-
// preserved across submits and deferred cleanup still releases
|
|
3603
|
-
// tracked buffers when GPU work completes. Profile runs keep the
|
|
3604
|
-
// sync path because resolveProfileTimings requires the submitted
|
|
3605
|
-
// work to be done.
|
|
3606
|
-
const chunkSubmitMode = resolvePrefillChunkSubmitMode(
|
|
3607
|
-
this.#state.runtimeConfig,
|
|
3608
|
-
this.#state.modelConfig
|
|
3609
|
-
);
|
|
3610
|
-
const chunkSubmitStart = performance.now();
|
|
3611
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3612
|
-
if (chunkSubmitMode === 'async' && !opts.profile) {
|
|
3613
|
-
currentRecorder.submit({ cleanup: 'deferred' });
|
|
3614
|
-
} else {
|
|
3615
|
-
await currentRecorder.submitAndWait();
|
|
3616
|
-
await recordProfile(currentRecorder);
|
|
3617
|
-
}
|
|
3618
|
-
prefillSubmitWaitMs += performance.now() - chunkSubmitStart;
|
|
3619
|
-
await traceActivationHealth(
|
|
3620
|
-
`PREFILL_LAYER_${l}_HEALTH`,
|
|
3621
|
-
currentHiddenBuffer,
|
|
3622
|
-
activationDtype,
|
|
3623
|
-
numTokens * config.hiddenSize
|
|
3624
|
-
);
|
|
3625
|
-
if (tracePrefillEnabled) {
|
|
3626
|
-
trace.perf('Prefill chunk submitted', {
|
|
3627
|
-
layer: l,
|
|
3628
|
-
elapsedMs: performance.now() - layerLoopTraceStart,
|
|
3629
|
-
prefillRecordMs,
|
|
3630
|
-
prefillSubmitWaitMs,
|
|
3631
|
-
});
|
|
3632
|
-
}
|
|
3633
|
-
currentRecorder = createRecorder('prefill-chunk');
|
|
3634
|
-
}
|
|
3635
|
-
}
|
|
3636
|
-
if (tracePrefillEnabled) {
|
|
3637
|
-
trace.perf('Prefill layer loop recorded', {
|
|
3638
|
-
numTokens,
|
|
3639
|
-
elapsedMs: performance.now() - layerLoopTraceStart,
|
|
3640
|
-
prefillRecordMs,
|
|
3641
|
-
prefillSubmitWaitMs,
|
|
3642
|
-
});
|
|
3643
|
-
}
|
|
3644
|
-
} finally {
|
|
3645
|
-
context.perLayerInputBuffer = null;
|
|
3646
|
-
if (perLayerInputs) {
|
|
3647
|
-
for (const buffer of perLayerInputs) {
|
|
3648
|
-
releasePerLayerInputBuffer(
|
|
3649
|
-
buffer,
|
|
3650
|
-
currentRecorder,
|
|
3651
|
-
context.decodeBuffers,
|
|
3652
|
-
this.#state.pleCache ?? null
|
|
3653
|
-
);
|
|
3654
|
-
}
|
|
3655
|
-
}
|
|
3656
|
-
releaseSharedAttentionState(context.sharedAttentionState, currentRecorder);
|
|
3657
|
-
this.#state.stats.prefillRecordMs = (this.#state.stats.prefillRecordMs ?? 0) + prefillRecordMs;
|
|
3658
|
-
this.#state.stats.prefillSubmitWaitMs = (this.#state.stats.prefillSubmitWaitMs ?? 0) + prefillSubmitWaitMs;
|
|
3659
|
-
}
|
|
3660
|
-
|
|
3661
|
-
{
|
|
3662
|
-
const tsirFixtureCfg = this.#state.operatorDiagnostics?.tsirFixture ?? null;
|
|
3663
|
-
if (tsirFixtureCfg && Array.isArray(tsirFixtureCfg.pendingReads) && tsirFixtureCfg.pendingReads.length > 0) {
|
|
3664
|
-
if (currentRecorder) {
|
|
3665
|
-
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3666
|
-
currentHiddenBuffer,
|
|
3667
|
-
currentRecorder,
|
|
3668
|
-
'prefill_tsir_drain_carry'
|
|
3669
|
-
);
|
|
3670
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3671
|
-
await currentRecorder.submitAndWait();
|
|
3672
|
-
await recordProfile(currentRecorder);
|
|
3673
|
-
currentRecorder = undefined;
|
|
3674
|
-
}
|
|
3675
|
-
await drainPendingTsirReads(tsirFixtureCfg);
|
|
3676
|
-
}
|
|
3677
|
-
}
|
|
3678
|
-
if (this.#state.finitenessBuffer) {
|
|
3679
|
-
if (currentRecorder) {
|
|
3680
|
-
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3681
|
-
currentHiddenBuffer,
|
|
3682
|
-
currentRecorder,
|
|
3683
|
-
'prefill_finiteness_carry'
|
|
3684
|
-
);
|
|
3685
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3686
|
-
await currentRecorder.submitAndWait();
|
|
3687
|
-
await recordProfile(currentRecorder);
|
|
3688
|
-
currentRecorder = undefined;
|
|
3689
|
-
}
|
|
3690
|
-
const isInfiniteData = await readBuffer(this.#state.finitenessBuffer, 16);
|
|
3691
|
-
const u32 = new Uint32Array(isInfiniteData.buffer, isInfiniteData.byteOffset, 4);
|
|
3692
|
-
const finitenessStatus = parseFinitenessStatusWords(u32, 0);
|
|
3693
|
-
if (finitenessStatus.triggered) {
|
|
3694
|
-
if (isGpuBufferInstance(currentHiddenBuffer)) {
|
|
3695
|
-
releaseBuffer(currentHiddenBuffer);
|
|
3696
|
-
}
|
|
3697
|
-
throw new FinitenessError(`F16 bounds exceeded during prefill${finitenessStatus.metadata}`);
|
|
3698
|
-
}
|
|
3699
|
-
}
|
|
3700
|
-
|
|
3701
|
-
if (benchmarkSubmits) {
|
|
3702
|
-
logSubmitStats(`Prefill (${numTokens} tokens, ${config.numLayers} layers)`);
|
|
3703
|
-
setTrackSubmits(false);
|
|
3704
|
-
}
|
|
3705
|
-
|
|
3706
|
-
if (opts.debug) {
|
|
3707
|
-
log.debug('Pipeline', `LAYER_LOOP_DONE, currentHiddenBuffer type=${currentHiddenBuffer?.constructor?.name}`);
|
|
3708
|
-
if (currentHiddenBuffer && allowReadback('pipeline.prefill.final-hidden')) {
|
|
3709
|
-
const lastTokenOffset = (numTokens - 1) * config.hiddenSize * activationBytes;
|
|
3710
|
-
const sampleSize = config.hiddenSize * activationBytes;
|
|
3711
|
-
const data = decodeReadback(
|
|
3712
|
-
await readBufferSlice(currentHiddenBuffer, lastTokenOffset, sampleSize),
|
|
3713
|
-
activationDtype
|
|
3714
|
-
);
|
|
3715
|
-
const nanCount = Array.from(data).filter(x => !Number.isFinite(x)).length;
|
|
3716
|
-
const nonZero = Array.from(data).filter(x => Number.isFinite(x) && x !== 0).slice(0, 5);
|
|
3717
|
-
log.debug('Pipeline', `FINAL_HIDDEN[pos=${numTokens - 1}]: nan=${nanCount}/${data.length}, sample=[${nonZero.map(x => x.toFixed(4)).join(', ')}]`);
|
|
3718
|
-
}
|
|
3719
|
-
}
|
|
3720
|
-
|
|
3721
|
-
if (hasGpuTimePrefill) {
|
|
3722
|
-
this.#state.stats.gpuTimePrefillMs = gpuTimePrefillMs;
|
|
3723
|
-
}
|
|
3724
|
-
|
|
3725
|
-
if (returnHidden) {
|
|
3726
|
-
if (currentRecorder) {
|
|
3727
|
-
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3728
|
-
currentHiddenBuffer,
|
|
3729
|
-
currentRecorder,
|
|
3730
|
-
'prefill_return_hidden_carry'
|
|
3731
|
-
);
|
|
3732
|
-
}
|
|
3733
|
-
return {
|
|
3734
|
-
numTokens,
|
|
3735
|
-
config,
|
|
3736
|
-
startPos,
|
|
3737
|
-
activationDtype,
|
|
3738
|
-
activationBytes,
|
|
3739
|
-
currentRecorder,
|
|
3740
|
-
recordProfile,
|
|
3741
|
-
debugCheckBuffer,
|
|
3742
|
-
currentHiddenBuffer,
|
|
3743
|
-
};
|
|
3744
|
-
}
|
|
3745
|
-
|
|
3746
|
-
|
|
3747
|
-
let lastLogits;
|
|
3748
|
-
let logitsVocabSize = config.vocabSize;
|
|
3749
|
-
let usedRecordedLogits = false;
|
|
3750
|
-
const lmHead = this.#state.weights.get('lm_head');
|
|
3751
|
-
const canRecordLogits = !!currentRecorder
|
|
3752
|
-
&& selectedLogitTokenIds === null
|
|
3753
|
-
&& !!lmHead
|
|
3754
|
-
&& !isCpuWeightBuffer(lmHead)
|
|
3755
|
-
&& !this.#state.disableRecordedLogits
|
|
3756
|
-
&& numTokens === 1;
|
|
3757
|
-
if (currentRecorder && canRecordLogits) {
|
|
3758
|
-
const logitsTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
3759
|
-
const recorded = await recordLogitsGPU(
|
|
3760
|
-
currentRecorder,
|
|
3761
|
-
currentHiddenBuffer,
|
|
3762
|
-
numTokens,
|
|
3763
|
-
getLogitsWeights(this.#state),
|
|
3764
|
-
getLogitsConfig(this.#state),
|
|
3765
|
-
this.#state.operatorDiagnostics
|
|
3766
|
-
);
|
|
3767
|
-
logitsVocabSize = recorded.vocabSize;
|
|
3768
|
-
usedRecordedLogits = true;
|
|
3769
|
-
|
|
3770
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3771
|
-
await currentRecorder.submitAndWait();
|
|
3772
|
-
await recordProfile(currentRecorder);
|
|
3773
|
-
|
|
3774
|
-
const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: recorded.logitsDtype });
|
|
3775
|
-
const lastLogitsSize = logitsVocabSize * logitsBytes;
|
|
3776
|
-
const lastLogitsOffset = (numTokens - 1) * lastLogitsSize;
|
|
3777
|
-
const logitsData = await readBufferSlice(recorded.logitsBuffer, lastLogitsOffset, lastLogitsSize);
|
|
3778
|
-
releaseBuffer(recorded.logitsBuffer);
|
|
3779
|
-
lastLogits = decodeReadback(logitsData, recorded.logitsDtype);
|
|
3780
|
-
if (tracePrefillEnabled) {
|
|
3781
|
-
trace.perf('Prefill recorded logits complete', {
|
|
3782
|
-
numTokens,
|
|
3783
|
-
vocabSize: logitsVocabSize,
|
|
3784
|
-
elapsedMs: performance.now() - logitsTraceStart,
|
|
3785
|
-
});
|
|
3786
|
-
}
|
|
3787
|
-
|
|
3788
|
-
const health = getLogitsHealth(lastLogits);
|
|
3789
|
-
if (health.nanCount > 0 || health.infCount > 0 || health.nonZeroCount === 0) {
|
|
3790
|
-
log.warn(
|
|
3791
|
-
'Logits',
|
|
3792
|
-
`Recorded logits invalid (nan=${health.nanCount} inf=${health.infCount} nonZero=${health.nonZeroCount}, maxAbs=${health.maxAbs.toFixed(3)}); recomputing without recorder.`
|
|
3793
|
-
);
|
|
3794
|
-
this.#state.disableRecordedLogits = true;
|
|
3795
|
-
this.#state.disableFusedDecode = true;
|
|
3796
|
-
const fallbackLogits = await computeLogits(
|
|
3797
|
-
currentHiddenBuffer,
|
|
3798
|
-
numTokens,
|
|
3799
|
-
getLogitsWeights(this.#state),
|
|
3800
|
-
getLogitsConfig(this.#state),
|
|
3801
|
-
this.#state.useGPU,
|
|
3802
|
-
this.#state.debugFlags,
|
|
3803
|
-
undefined,
|
|
3804
|
-
debugCheckBuffer,
|
|
3805
|
-
this.#state.runtimeConfig.shared.debug.probes,
|
|
3806
|
-
{ lastPositionOnly: true },
|
|
3807
|
-
this.#state.operatorDiagnostics
|
|
3808
|
-
);
|
|
3809
|
-
const fallbackHealth = getLogitsHealth(fallbackLogits);
|
|
3810
|
-
if (fallbackHealth.nanCount > 0 || fallbackHealth.infCount > 0 || fallbackHealth.nonZeroCount === 0) {
|
|
3811
|
-
throw new Error(
|
|
3812
|
-
`[Logits] Fallback logits invalid (nan=${fallbackHealth.nanCount} inf=${fallbackHealth.infCount} nonZero=${fallbackHealth.nonZeroCount}, maxAbs=${fallbackHealth.maxAbs.toFixed(3)}). ` +
|
|
3813
|
-
'This indicates upstream kernel output is NaN/Inf (often prefill attention/matmul).'
|
|
3814
|
-
);
|
|
3815
|
-
}
|
|
3816
|
-
logitsVocabSize = config.vocabSize;
|
|
3817
|
-
usedRecordedLogits = false;
|
|
3818
|
-
lastLogits = fallbackLogits.length === logitsVocabSize
|
|
3819
|
-
? fallbackLogits
|
|
3820
|
-
: extractLastPositionLogits(fallbackLogits, numTokens, logitsVocabSize);
|
|
3821
|
-
}
|
|
3822
|
-
|
|
3823
|
-
releaseBuffer(currentHiddenBuffer);
|
|
3824
|
-
} else {
|
|
3825
|
-
const logitsTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
3826
|
-
if (currentRecorder) {
|
|
3827
|
-
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3828
|
-
currentHiddenBuffer,
|
|
3829
|
-
currentRecorder,
|
|
3830
|
-
'prefill_logits_carry'
|
|
3831
|
-
);
|
|
3832
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3833
|
-
await currentRecorder.submitAndWait();
|
|
3834
|
-
await recordProfile(currentRecorder);
|
|
3835
|
-
}
|
|
3836
|
-
const logits = await computeLogits(
|
|
3837
|
-
currentHiddenBuffer,
|
|
3838
|
-
numTokens,
|
|
3839
|
-
getLogitsWeights(this.#state),
|
|
3840
|
-
getLogitsConfig(this.#state),
|
|
3841
|
-
this.#state.useGPU,
|
|
3842
|
-
this.#state.debugFlags,
|
|
3843
|
-
undefined,
|
|
3844
|
-
debugCheckBuffer,
|
|
3845
|
-
this.#state.runtimeConfig.shared.debug.probes,
|
|
3846
|
-
{ lastPositionOnly: true, selectedTokenIds: selectedLogitTokenIds },
|
|
3847
|
-
this.#state.operatorDiagnostics
|
|
3848
|
-
);
|
|
3849
|
-
|
|
3850
|
-
if (selectedLogitTokenIds !== null) {
|
|
3851
|
-
lastLogits = logits;
|
|
3852
|
-
} else {
|
|
3853
|
-
lastLogits = logits.length === logitsVocabSize
|
|
3854
|
-
? logits
|
|
3855
|
-
: extractLastPositionLogits(logits, numTokens, logitsVocabSize);
|
|
3856
|
-
}
|
|
3857
|
-
releaseBuffer(currentHiddenBuffer);
|
|
3858
|
-
if (tracePrefillEnabled) {
|
|
3859
|
-
trace.perf('Prefill logits complete', {
|
|
3860
|
-
numTokens,
|
|
3861
|
-
vocabSize: logitsVocabSize,
|
|
3862
|
-
elapsedMs: performance.now() - logitsTraceStart,
|
|
3863
|
-
});
|
|
3864
|
-
}
|
|
3865
|
-
}
|
|
3866
|
-
|
|
3867
|
-
this.#state.currentSeqLen = startPos + numTokens;
|
|
3868
|
-
|
|
3869
|
-
if (usedRecordedLogits) {
|
|
3870
|
-
if (logitsVocabSize < config.vocabSize) {
|
|
3871
|
-
const padded = new Float32Array(config.vocabSize);
|
|
3872
|
-
padded.set(lastLogits);
|
|
3873
|
-
padded.fill(-Infinity, logitsVocabSize);
|
|
3874
|
-
lastLogits = padded;
|
|
3875
|
-
}
|
|
3876
|
-
if (config.finalLogitSoftcapping != null) {
|
|
3877
|
-
applySoftcapping(lastLogits, config.finalLogitSoftcapping);
|
|
3878
|
-
}
|
|
3879
|
-
}
|
|
3880
|
-
|
|
3881
|
-
if (opts.debug && selectedLogitTokenIds === null) {
|
|
3882
|
-
logitsSanity(lastLogits, 'Prefill', (tokens) => resolveTokenText(this.#state.tokenizer, tokens));
|
|
3883
|
-
}
|
|
3884
|
-
if (isTraceEnabled('logits')) {
|
|
3885
|
-
trace.logits(selectedLogitTokenIds === null ? 'PREFILL_LOGITS_HEALTH' : 'PREFILL_SELECTED_LOGITS_HEALTH', getLogitsHealth(lastLogits));
|
|
3886
|
-
}
|
|
3887
|
-
|
|
3888
|
-
if (opts.debug) {
|
|
3889
|
-
if (this.#state.kvCache?.hasGPUCache?.()) {
|
|
3890
|
-
log.debug('Pipeline', `KV cache active after prefill: seqLen=${this.#state.kvCache.getKeyCache(0)?.constructor.name ?? '?'}`);
|
|
3891
|
-
} else {
|
|
3892
|
-
log.warn('Pipeline', `KV cache NOT active after prefill! hasGPUCache=${this.#state.kvCache?.hasGPUCache?.()}`);
|
|
3893
|
-
}
|
|
3894
|
-
}
|
|
3895
|
-
|
|
3896
|
-
if (tracePrefillEnabled) {
|
|
3897
|
-
trace.perf('Prefill phase complete', {
|
|
3898
|
-
numTokens,
|
|
3899
|
-
totalMs: performance.now() - prefillTraceStart,
|
|
3900
|
-
});
|
|
3901
|
-
}
|
|
3902
|
-
|
|
3903
|
-
return lastLogits;
|
|
581
|
+
return _prefillImpl.apply(this, arguments);
|
|
3904
582
|
}
|
|
3905
583
|
|
|
3906
|
-
|
|
3907
584
|
async _decodeStep(currentIds, opts) {
|
|
3908
|
-
|
|
3909
|
-
const stepResult = await this._decodeStepToLogits(currentIds, opts);
|
|
3910
|
-
return this._sampleNextTokenFromLogits(stepResult.logits, currentIds, opts);
|
|
3911
|
-
}
|
|
3912
|
-
const debugCheckBuffer = this.#state.debug
|
|
3913
|
-
? (buffer, label, numTokens, expectedDim) =>
|
|
3914
|
-
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
3915
|
-
: undefined;
|
|
3916
|
-
return decodeStep(this.#state, currentIds, opts, this._getDecodeHelpers(debugCheckBuffer));
|
|
585
|
+
return _decodeStepImpl.apply(this, arguments);
|
|
3917
586
|
}
|
|
3918
587
|
|
|
3919
588
|
async decodeStepLogits(currentIds, options = {}) {
|
|
3920
|
-
|
|
3921
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
3922
|
-
throw new Error('Generation already in progress');
|
|
3923
|
-
}
|
|
3924
|
-
resetActiveExecutionPlan(this.#state);
|
|
3925
|
-
|
|
3926
|
-
validateCallTimeOptions(options);
|
|
3927
|
-
|
|
3928
|
-
const opts = this._resolveStepOptions(options);
|
|
3929
|
-
return this._decodeStepToLogits(currentIds, opts);
|
|
589
|
+
return decodeStepLogitsImpl.apply(this, arguments);
|
|
3930
590
|
}
|
|
3931
591
|
|
|
3932
592
|
async advanceWithToken(tokenId, options = {}) {
|
|
3933
|
-
|
|
3934
|
-
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
3935
|
-
resetActiveExecutionPlan(this.#state);
|
|
3936
|
-
assertIncrementalDecodeSupport(this.#state, 'advanceWithToken');
|
|
3937
|
-
|
|
3938
|
-
validateCallTimeOptions(options);
|
|
3939
|
-
|
|
3940
|
-
const opts = this._resolveStepOptions(options);
|
|
3941
|
-
const debugCheckBuffer = this.#state.debug
|
|
3942
|
-
? (buffer, label, numTokens, expectedDim) =>
|
|
3943
|
-
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
3944
|
-
: undefined;
|
|
3945
|
-
|
|
3946
|
-
this._assertTokenIdInRange(tokenId, 'advanceWithToken');
|
|
3947
|
-
await advanceWithToken(this.#state, tokenId, opts, this._getDecodeHelpers(debugCheckBuffer));
|
|
593
|
+
return advanceWithTokenImpl.apply(this, arguments);
|
|
3948
594
|
}
|
|
3949
595
|
|
|
3950
596
|
async advanceWithTokenAndEmbedding(tokenId, options = {}) {
|
|
3951
|
-
|
|
3952
|
-
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
3953
|
-
resetActiveExecutionPlan(this.#state);
|
|
3954
|
-
assertIncrementalDecodeSupport(this.#state, 'advanceWithTokenAndEmbedding');
|
|
3955
|
-
|
|
3956
|
-
validateCallTimeOptions(options);
|
|
3957
|
-
|
|
3958
|
-
const opts = this._resolveStepOptions(options);
|
|
3959
|
-
const embeddingMode = resolveAdvanceEmbeddingMode(this.#state, options);
|
|
3960
|
-
const debugCheckBuffer = this.#state.debug
|
|
3961
|
-
? (buffer, label, numTokens, expectedDim) =>
|
|
3962
|
-
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
3963
|
-
: undefined;
|
|
3964
|
-
|
|
3965
|
-
this._assertTokenIdInRange(tokenId, 'advanceWithTokenAndEmbedding');
|
|
3966
|
-
return runAdvanceWithTokenAndEmbedding(
|
|
3967
|
-
this.#state,
|
|
3968
|
-
tokenId,
|
|
3969
|
-
opts,
|
|
3970
|
-
this._getDecodeHelpers(debugCheckBuffer),
|
|
3971
|
-
embeddingMode
|
|
3972
|
-
);
|
|
597
|
+
return advanceWithTokenAndEmbeddingImpl.apply(this, arguments);
|
|
3973
598
|
}
|
|
3974
599
|
|
|
3975
600
|
async _generateNTokensGPU(startToken, N, currentIds, opts) {
|
|
3976
|
-
|
|
3977
|
-
? (buffer, label, numTokens, expectedDim) =>
|
|
3978
|
-
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
3979
|
-
: undefined;
|
|
3980
|
-
return generateNTokensGPU(this.#state, startToken, N, currentIds, opts, this._getDecodeHelpers(debugCheckBuffer));
|
|
601
|
+
return _generateNTokensGPUImpl.apply(this, arguments);
|
|
3981
602
|
}
|
|
3982
603
|
}
|