doppler-gpu 0.5.1 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -577
- package/LICENSE +21 -201
- package/README.md +165 -222
- package/models/catalog.json +241 -33
- package/package.json +106 -87
- package/src/capsule-runtime.d.ts +25 -0
- package/src/capsule-runtime.js +24 -0
- package/src/capsule.d.ts +8 -0
- package/src/capsule.js +4 -0
- package/src/cli/argument-contract.js +200 -0
- package/src/cli/browser-report-output.js +24 -0
- package/src/cli/cli-model-resolution.js +2 -2
- package/src/cli/cli-output.js +10 -0
- package/src/cli/commands/bundle.js +828 -0
- package/src/cli/doppler-cli.js +50 -1457
- package/src/cli/doppler-serve.js +18 -244
- package/src/cli/output/manifest.js +368 -0
- package/src/cli/runtime-profiles.js +1 -1
- package/src/cli/serve/capsule-handler.d.ts +33 -0
- package/src/cli/serve/capsule-handler.js +105 -0
- package/src/cli/serve/capsule-http.js +102 -0
- package/src/cli/serve/receipt.js +269 -0
- package/src/client/capsule-host.browser.js +2 -0
- package/src/client/capsule-host.d.ts +5 -0
- package/src/client/capsule-host.js +2 -0
- package/src/client/doppler-api.browser.js +42 -0
- package/src/client/doppler-api.d.ts +38 -1
- package/src/client/doppler-api.js +41 -1
- package/src/client/doppler-registry.js +15 -7
- package/src/client/electron/index.d.ts +4 -0
- package/src/client/electron/index.js +13 -0
- package/src/client/electron/ipc-contract.d.ts +9 -0
- package/src/client/electron/ipc-contract.js +56 -0
- package/src/client/electron/release-state.d.ts +53 -0
- package/src/client/electron/release-state.js +425 -0
- package/src/client/electron/renderer-runtime.d.ts +19 -0
- package/src/client/electron/renderer-runtime.js +104 -0
- package/src/client/failure-taxonomy.js +1 -28
- package/src/client/fault-injection.js +0 -8
- package/src/client/inspection.js +1 -1
- package/src/client/model-host/index.d.ts +140 -0
- package/src/client/model-host/index.js +548 -0
- package/src/client/model-host/model-evidence.js +255 -0
- package/src/client/model-host/model-session.d.ts +323 -0
- package/src/client/{runtime → model-host}/model-session.js +202 -148
- package/src/client/provider.d.ts +18 -2
- package/src/client/provider.js +26 -3
- package/src/client/receipt.js +30 -72
- package/src/client/runtime/capsule-acquisition.d.ts +18 -0
- package/src/client/runtime/capsule-acquisition.js +103 -0
- package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
- package/src/client/runtime/capsule-adapter-execution.js +63 -0
- package/src/client/runtime/capsule-artifact-source.js +62 -0
- package/src/client/runtime/capsule-embedding.d.ts +29 -0
- package/src/client/runtime/capsule-embedding.js +76 -0
- package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
- package/src/client/runtime/capsule-forecast-program.js +85 -0
- package/src/client/runtime/capsule-forecast.d.ts +17 -0
- package/src/client/runtime/capsule-forecast.js +31 -0
- package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
- package/src/client/runtime/capsule-operation-adapters.js +69 -0
- package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
- package/src/client/runtime/capsule-operation-executor.js +101 -0
- package/src/client/runtime/capsule-program-adapter.js +153 -0
- package/src/client/runtime/capsule-release-authorization.js +25 -0
- package/src/client/runtime/capsule-rerank.d.ts +50 -0
- package/src/client/runtime/capsule-rerank.js +105 -0
- package/src/client/runtime/capsule-session-execution.js +89 -0
- package/src/client/runtime/command-executor.d.ts +15 -0
- package/src/client/runtime/command-executor.js +128 -0
- package/src/client/runtime/composition-root.d.ts +77 -0
- package/src/client/runtime/composition-root.js +352 -0
- package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
- package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
- package/src/client/runtime/index.d.ts +1 -117
- package/src/client/runtime/index.js +1 -433
- package/src/client/runtime/lora.d.ts +14 -0
- package/src/client/runtime/lora.js +19 -1
- package/src/client/runtime/model-manager.js +12 -0
- package/src/client/runtime/model-rerank.js +30 -0
- package/src/client/runtime/model-session.d.ts +1 -221
- package/src/client/runtime/model-source.d.ts +9 -0
- package/src/client/runtime/model-source.js +46 -17
- package/src/client/runtime/node-quickstart-cache.js +1 -1
- package/src/client/runtime/resolution-policy.d.ts +20 -0
- package/src/client/runtime/resolution-policy.js +84 -0
- package/src/client/runtime/resource-binder.d.ts +24 -0
- package/src/client/runtime/resource-binder.js +135 -0
- package/src/client/runtime/scoped-session.d.ts +10 -1
- package/src/client/runtime/scoped-session.js +20 -1
- package/src/client/runtime/session-controller.d.ts +19 -0
- package/src/client/runtime/session-controller.js +92 -0
- package/src/client/runtime/source-runtime.js +1 -1
- package/src/client/runtime/target-selector.d.ts +9 -0
- package/src/client/runtime/target-selector.js +2 -0
- package/src/client/runtime/types.d.ts +1 -0
- package/src/client/runtime/verified-capsule-artifact-store.js +101 -0
- package/src/client/wrap-pipeline-handle.js +3 -9
- package/src/config/backward-registry-loader.js +1 -1
- package/src/config/capsule-adapter-policy.js +30 -0
- package/src/config/capsule-adapters.d.ts +21 -0
- package/src/config/capsule-adapters.js +40 -0
- package/src/config/capsule-adapters.json +9 -0
- package/src/config/capsule-loading.d.ts +2 -0
- package/src/config/capsule-loading.js +14 -0
- package/src/config/capsule-loading.schema.json +12 -0
- package/src/config/capsule-operation.d.ts +19 -0
- package/src/config/capsule-operation.js +63 -0
- package/src/config/capsule-operations.json +17 -0
- package/src/config/capsule-release-contract.d.ts +66 -0
- package/src/config/capsule-release-contract.js +197 -0
- package/src/config/capsule-release-events.d.ts +53 -0
- package/src/config/capsule-release-events.js +172 -0
- package/src/config/capsule-sequence-receipt.js +19 -0
- package/src/config/capsule-serve.d.ts +9 -0
- package/src/config/capsule-serve.js +28 -0
- package/src/config/capsule-serve.schema.json +14 -0
- package/src/config/capsule-signature.d.ts +6 -0
- package/src/config/capsule-signature.js +50 -0
- package/src/config/capsule-v2.d.ts +70 -0
- package/src/config/capsule-v2.js +431 -0
- package/src/config/capsule-v3.d.ts +19 -0
- package/src/config/capsule-v3.js +72 -0
- package/src/config/capsule.d.ts +22 -0
- package/src/config/capsule.js +52 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +62 -4
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +6 -6
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +6 -6
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +3 -3
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +6 -6
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +6 -6
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +6 -6
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +7 -7
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +6 -6
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +60 -8
- package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
- package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
- package/src/config/dtype-transition-contract.js +40 -0
- package/src/config/embedding-contract.js +26 -0
- package/src/config/embedding-reference.js +85 -0
- package/src/config/evidence/source-intake-policy.json +1 -1
- package/src/config/execution-contract-check.d.ts +16 -73
- package/src/config/execution-contract-check.js +7 -429
- package/src/config/forecast-manifest.js +56 -0
- package/src/config/generation-contract.d.ts +39 -0
- package/src/config/generation-contract.js +66 -0
- package/src/config/generation-contract.json +32 -0
- package/src/config/heterogeneous-attention-contract.js +46 -0
- package/src/config/initial-execution-identity.d.ts +58 -0
- package/src/config/initial-execution-identity.js +331 -0
- package/src/config/kernel-path-loader.d.ts +17 -0
- package/src/config/kernel-path-loader.js +42 -8
- package/src/config/kernel-registry-contract.d.ts +25 -0
- package/src/config/kernel-registry-contract.js +46 -0
- package/src/config/kernels/kernel-ref-digests.js +66 -18
- package/src/config/kernels/registry.js +1 -1
- package/src/config/kernels/registry.json +13617 -11377
- package/src/config/lora-layouts.d.ts +10 -0
- package/src/config/lora-layouts.js +17 -0
- package/src/config/lora-layouts.json +9 -0
- package/src/config/manifest-inference-validation.js +575 -0
- package/src/config/merge/execution-patches.js +12 -0
- package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
- package/src/config/merge/value-selection.js +38 -0
- package/src/config/merge.js +43 -11
- package/src/config/model-ir-v2.d.ts +57 -0
- package/src/config/model-ir-v2.js +258 -0
- package/src/config/model-ir.d.ts +49 -0
- package/src/config/model-ir.js +180 -0
- package/src/config/param-validator.js +1 -1
- package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
- package/src/config/platforms/loader.js +2 -1
- package/src/config/production-release-evidence.d.ts +23 -0
- package/src/config/production-release-evidence.js +423 -0
- package/src/config/production-release.js +321 -0
- package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
- package/src/config/required-inference-fields-contract-check.js +2 -2
- package/src/config/rerank-reference.d.ts +28 -0
- package/src/config/rerank-reference.js +89 -0
- package/src/config/revocation-policy.d.ts +65 -0
- package/src/config/revocation-policy.js +290 -0
- package/src/config/revocation-registry.json +11 -0
- package/src/config/revocation-updates.d.ts +68 -0
- package/src/config/revocation-updates.js +440 -0
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
- package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
- package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
- package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
- package/src/config/runtime-merge.js +1 -1
- package/src/config/runtime.js +1 -1
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/conversion-report.schema.d.ts +3 -0
- package/src/config/schema/conversion-report.schema.js +29 -0
- package/src/config/schema/converter.schema.d.ts +1 -0
- package/src/config/schema/debug.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +6 -5
- package/src/config/schema/emulation.schema.js +1 -1
- package/src/config/schema/execution-v1.schema.d.ts +3 -0
- package/src/config/schema/execution-v1.schema.js +19 -0
- package/src/config/schema/harness.schema.d.ts +13 -0
- package/src/config/schema/harness.schema.js +6 -0
- package/src/config/schema/inference.schema.d.ts +21 -2
- package/src/config/schema/kernel-thresholds.schema.js +1 -1
- package/src/config/schema/manifest.schema.d.ts +26 -7
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/program-bundle.schema.d.ts +52 -6
- package/src/config/schema/program-bundle.schema.js +145 -10
- package/src/config/sequence-reference.d.ts +26 -0
- package/src/config/sequence-reference.js +100 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
- package/src/config/target-plan.d.ts +108 -0
- package/src/config/target-plan.js +411 -0
- package/src/config/training-defaults.js +1 -1
- package/src/config/transforms/capability-transform-resolver.js +4 -105
- package/src/config/transforms/execution-graph/dtype.js +591 -0
- package/src/config/transforms/execution-graph/fusion.js +436 -0
- package/src/config/transforms/execution-graph/validation.js +414 -0
- package/src/config/transforms/execution-graph-transforms.js +13 -1419
- package/src/converter/artifact-identity.js +475 -0
- package/src/converter/conversion-plan.js +1 -1
- package/src/converter/core.js +94 -2096
- package/src/converter/forge-candidate-evaluation.js +271 -0
- package/src/converter/forge-qualification.js +163 -0
- package/src/converter/forge-stages.js +800 -0
- package/src/converter/manifest-builder.js +683 -0
- package/src/converter/quantization-info.js +4 -1
- package/src/converter/quantizer.js +76 -13
- package/src/converter/shard-packer.d.ts +1 -1
- package/src/converter/shard-packer.js +2 -2
- package/src/converter/tensor-transform.js +640 -0
- package/src/converter/tokenizer-contract.js +243 -0
- package/src/converter/tokenizer-manifest.js +243 -0
- package/src/debug/history.js +0 -8
- package/src/debug/index.js +0 -10
- package/src/debug/tensor.js +1 -1
- package/src/debug/trace.js +0 -7
- package/src/experimental/adapters/adapter-manager.js +2 -0
- package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
- package/src/experimental/adapters/adapter-manifest.js +11 -0
- package/src/experimental/adapters/artifact-contract.d.ts +2 -2
- package/src/experimental/adapters/artifact-contract.js +5 -2
- package/src/experimental/adapters/lora-loader.d.ts +10 -1
- package/src/experimental/adapters/lora-loader.js +110 -48
- package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
- package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
- package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
- package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
- package/src/experimental/browser/browser-converter.js +11 -11
- package/src/experimental/diffusion/index.d.ts +1 -1
- package/src/experimental/diffusion/index.js +1 -1
- package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
- package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
- package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
- package/src/experimental/diffusion/vae.js +596 -0
- package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
- package/src/experimental/distribution/shard-delivery/retry.js +392 -0
- package/src/experimental/distribution/shard-delivery/transport.js +831 -0
- package/src/experimental/distribution/shard-delivery.js +4 -1709
- package/src/experimental/energy/index.d.ts +2 -2
- package/src/experimental/energy/index.js +2 -2
- package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
- package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
- package/src/experimental/logits/cpu-output.d.ts +27 -0
- package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
- package/src/experimental/logits/cpu.d.ts +87 -0
- package/src/experimental/logits/cpu.js +95 -0
- package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
- package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
- package/src/experimental/orchestration/index.d.ts +1 -1
- package/src/experimental/orchestration/index.js +1 -1
- package/src/experimental/training/artifacts.js +2 -2
- package/src/experimental/training/checkpoint.js +2 -2
- package/src/experimental/training/distillation/artifacts.js +1 -1
- package/src/experimental/training/distillation/dataset.js +2 -2
- package/src/experimental/training/distillation/eval.js +1 -1
- package/src/experimental/training/distillation/runtime.js +2 -2
- package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
- package/src/experimental/training/distillation/student-fixture/model.js +353 -0
- package/src/experimental/training/distillation/student-fixture.js +2 -469
- package/src/experimental/training/lora/checkpoint.js +746 -0
- package/src/experimental/training/lora/execution.js +550 -0
- package/src/experimental/training/lora/plan.js +387 -0
- package/src/experimental/training/lora/recovery.js +418 -0
- package/src/experimental/training/lora-pipeline.js +7 -1949
- package/src/experimental/training/operator-artifacts.js +2 -2
- package/src/experimental/training/operator-command.js +4 -4
- package/src/experimental/training/runner/evaluation.js +62 -0
- package/src/experimental/training/runner/lifecycle.js +257 -0
- package/src/experimental/training/runner.js +5 -268
- package/src/experimental/training/suite/execution.js +879 -0
- package/src/experimental/training/suite/plan.js +855 -0
- package/src/experimental/training/suite.js +5 -1659
- package/src/experimental/training/workloads.d.ts +5 -5
- package/src/experimental/training/workloads.js +6 -6
- package/src/experimental/turboquant/codebook.js +470 -0
- package/src/formats/quantization-constants.d.ts +11 -0
- package/src/formats/quantization-constants.js +11 -0
- package/src/formats/rdrr/integrity.js +1 -1
- package/src/formats/rdrr/lowerings.js +0 -29
- package/src/formats/rdrr/merkle.js +1 -1
- package/src/formats/rdrr/tensor-config-validator.js +25 -5
- package/src/formats/rdrr/validation.js +3 -3
- package/src/gpu/command-recorder.d.ts +10 -1
- package/src/gpu/command-recorder.js +58 -14
- package/src/gpu/constant-buffer.js +29 -0
- package/src/gpu/device-state.js +44 -0
- package/src/gpu/device.js +11 -41
- package/src/gpu/kernel-runtime.js +2 -1
- package/src/gpu/kernels/activation-static-qdq.js +1 -1
- package/src/gpu/kernels/append_embedding.wgsl +14 -0
- package/src/gpu/kernels/attention/executors.js +960 -0
- package/src/gpu/kernels/attention/plan.js +472 -0
- package/src/gpu/kernels/attention.d.ts +4 -0
- package/src/gpu/kernels/attention.js +145 -1545
- package/src/gpu/kernels/attention_relative.wgsl +59 -0
- package/src/gpu/kernels/backward/adam.js +2 -1
- package/src/gpu/kernels/backward/attention_backward.js +1 -1
- package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
- package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
- package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
- package/src/gpu/kernels/backward/embed_backward.js +2 -1
- package/src/gpu/kernels/backward/executor.d.ts +34 -0
- package/src/gpu/kernels/backward/executor.js +158 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
- package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/gelu_backward.js +1 -1
- package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
- package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
- package/src/gpu/kernels/backward/matmul.js +133 -0
- package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
- package/src/gpu/kernels/backward/matmul_backward.js +1 -1
- package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
- package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
- package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/rope_backward.js +2 -1
- package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/scale_backward.js +2 -1
- package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/silu_backward.js +1 -1
- package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/softmax_backward.js +1 -1
- package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
- package/src/gpu/kernels/backward/utils.js +2 -392
- package/src/gpu/kernels/bind-group-validation.js +16 -0
- package/src/gpu/kernels/cast.js +2 -1
- package/src/gpu/kernels/check-finiteness.js +13 -49
- package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
- package/src/gpu/kernels/check-stop.js +14 -33
- package/src/gpu/kernels/check_finiteness.wgsl +33 -0
- package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
- package/src/gpu/kernels/check_stop.wgsl +20 -0
- package/src/gpu/kernels/clamp.js +1 -1
- package/src/gpu/kernels/conv2d.js +1 -1
- package/src/gpu/kernels/cross_entropy_loss.js +1 -1
- package/src/gpu/kernels/depthwise_conv1d.js +1 -1
- package/src/gpu/kernels/depthwise_conv2d.js +1 -1
- package/src/gpu/kernels/dequant.js +7 -2
- package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
- package/src/gpu/kernels/dispatch.js +12 -1
- package/src/gpu/kernels/embedding-pool.js +72 -0
- package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
- package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
- package/src/gpu/kernels/energy.js +2 -1
- package/src/gpu/kernels/forecast_output.wgsl +17 -0
- package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
- package/src/gpu/kernels/fused_ffn.js +10 -220
- package/src/gpu/kernels/fused_matmul_residual.js +2 -1
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
- package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
- package/src/gpu/kernels/gated-short-conv.js +11 -69
- package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
- package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
- package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
- package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
- package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
- package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
- package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
- package/src/gpu/kernels/gather.d.ts +1 -1
- package/src/gpu/kernels/gather.js +30 -6
- package/src/gpu/kernels/gather_bf16.wgsl +69 -0
- package/src/gpu/kernels/gelu.d.ts +4 -0
- package/src/gpu/kernels/gelu.js +12 -2
- package/src/gpu/kernels/gelu.wgsl +16 -0
- package/src/gpu/kernels/gelu_f16.wgsl +16 -0
- package/src/gpu/kernels/gemma4-route-expert.js +1 -1
- package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
- package/src/gpu/kernels/groupnorm.js +1 -1
- package/src/gpu/kernels/index.d.ts +38 -5
- package/src/gpu/kernels/index.js +18 -4
- package/src/gpu/kernels/instance_stats.wgsl +26 -0
- package/src/gpu/kernels/kernel-base.js +1 -1
- package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
- package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
- package/src/gpu/kernels/kernel-configs.d.ts +10 -61
- package/src/gpu/kernels/kernel-configs.js +26 -54
- package/src/gpu/kernels/kernel-execution.js +141 -0
- package/src/gpu/kernels/kv-cache-write.js +4 -4
- package/src/gpu/kernels/kv-quantize.js +4 -1
- package/src/gpu/kernels/l2-normalize.js +36 -0
- package/src/gpu/kernels/l2_normalize.wgsl +53 -0
- package/src/gpu/kernels/layernorm.d.ts +4 -4
- package/src/gpu/kernels/layernorm.js +44 -7
- package/src/gpu/kernels/linear-attention-core.js +29 -449
- package/src/gpu/kernels/linear_attention.js +1 -1
- package/src/gpu/kernels/lm-head-argmax.js +2 -2
- package/src/gpu/kernels/lm-head-select-logits.js +2 -2
- package/src/gpu/kernels/logit-finalize.js +103 -0
- package/src/gpu/kernels/logit-merge.js +13 -125
- package/src/gpu/kernels/logit-scatter.js +40 -0
- package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
- package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
- package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
- package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
- package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
- package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
- package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
- package/src/gpu/kernels/matmul-dispatch.js +6 -1
- package/src/gpu/kernels/matmul-selection.js +3 -2
- package/src/gpu/kernels/matmul.js +4 -15
- package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
- package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
- package/src/gpu/kernels/modulate.js +1 -1
- package/src/gpu/kernels/moe.js +3 -1
- package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
- package/src/gpu/kernels/patch_context.wgsl +29 -0
- package/src/gpu/kernels/pipeline-cache.js +3 -1
- package/src/gpu/kernels/pixel_shuffle.js +1 -1
- package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
- package/src/gpu/kernels/relu.js +1 -1
- package/src/gpu/kernels/rep-penalty.js +30 -5
- package/src/gpu/kernels/rep_penalty.wgsl +25 -30
- package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
- package/src/gpu/kernels/repeat_channels.js +1 -1
- package/src/gpu/kernels/residual.js +2 -2
- package/src/gpu/kernels/rmsnorm-pair.js +1 -1
- package/src/gpu/kernels/rmsnorm-qk.js +1 -1
- package/src/gpu/kernels/rmsnorm-stats.js +14 -113
- package/src/gpu/kernels/rmsnorm.js +1 -1
- package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
- package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
- package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
- package/src/gpu/kernels/rope-precompute.js +160 -0
- package/src/gpu/kernels/rope-qk.js +1 -1
- package/src/gpu/kernels/rope.js +2 -2
- package/src/gpu/kernels/rope_precompute.wgsl +74 -0
- package/src/gpu/kernels/sample.d.ts +1 -0
- package/src/gpu/kernels/sample.js +36 -17
- package/src/gpu/kernels/sample.wgsl +112 -157
- package/src/gpu/kernels/sample_f16.wgsl +168 -162
- package/src/gpu/kernels/scale.js +1 -1
- package/src/gpu/kernels/shader-cache.js +8 -4
- package/src/gpu/kernels/shader-source-scope.js +68 -0
- package/src/gpu/kernels/silu.js +2 -1
- package/src/gpu/kernels/soft-embedding.js +3 -1
- package/src/gpu/kernels/softmax.js +4 -2
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
- package/src/gpu/kernels/split_qg.js +1 -1
- package/src/gpu/kernels/split_qkv.d.ts +2 -0
- package/src/gpu/kernels/split_qkv.js +18 -3
- package/src/gpu/kernels/split_qkv.wgsl +5 -2
- package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
- package/src/gpu/kernels/submit_probe.wgsl +6 -0
- package/src/gpu/kernels/transpose.js +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +4 -535
- package/src/gpu/kernels/uniform-utils.js +7 -2
- package/src/gpu/kernels/upsample2d.js +1 -1
- package/src/gpu/kernels/utils.js +9 -330
- package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
- package/src/gpu/kernels/vision-average-pool.js +68 -0
- package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
- package/src/gpu/kernels/vision-patch-embed.js +117 -0
- package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
- package/src/gpu/kernels/vision-position-embedding.js +79 -0
- package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
- package/src/gpu/kernels/vision-rope-2d.js +73 -0
- package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
- package/src/gpu/kernels/vision-spatial-merge.js +100 -0
- package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
- package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
- package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
- package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
- package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
- package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
- package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
- package/src/gpu/readback-buffer.js +20 -0
- package/src/gpu/submit-probe.js +2 -26
- package/src/gpu/weight-buffer.d.ts +1 -0
- package/src/gpu/weight-buffer.js +7 -0
- package/src/index.d.ts +12 -0
- package/src/index.js +2 -0
- package/src/inference/browser-harness/benchmark-execution.js +872 -0
- package/src/inference/browser-harness/benchmark-rerank.js +284 -0
- package/src/inference/browser-harness/decode-diagnostics.js +241 -0
- package/src/inference/browser-harness/diffusion-performance.js +102 -0
- package/src/inference/browser-harness/inference-execution.js +543 -0
- package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
- package/src/inference/browser-harness/model-resolution.js +137 -0
- package/src/inference/browser-harness/report.js +338 -0
- package/src/inference/browser-harness/request.js +263 -0
- package/src/inference/browser-harness/runtime-config.d.ts +25 -0
- package/src/inference/browser-harness/runtime-config.js +255 -0
- package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
- package/src/inference/browser-harness/runtime-isolation.js +45 -0
- package/src/inference/browser-harness/runtime-report.d.ts +26 -0
- package/src/inference/browser-harness/runtime-report.js +117 -0
- package/src/inference/browser-harness/suite-summary.js +39 -0
- package/src/inference/browser-harness/text-evidence.js +694 -0
- package/src/inference/browser-harness/text-execution.js +757 -0
- package/src/inference/browser-harness/text-input.js +725 -0
- package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
- package/src/inference/browser-harness-contract-helpers.js +1 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
- package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
- package/src/inference/browser-harness-runtime-helpers.js +3 -423
- package/src/inference/browser-harness-text-helpers.js +31 -2199
- package/src/inference/browser-harness.d.ts +15 -2
- package/src/inference/browser-harness.js +38 -1952
- package/src/inference/kv-cache/base.js +67 -381
- package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
- package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
- package/src/inference/kv-cache/quantized.d.ts +2 -2
- package/src/inference/kv-cache/quantized.js +4 -8
- package/src/inference/kv-cache/update.js +309 -0
- package/src/inference/model-checkpoint-evidence.d.ts +75 -0
- package/src/inference/model-checkpoint-evidence.js +264 -0
- package/src/inference/moe-router.js +36 -216
- package/src/inference/multi-model-network.js +5 -341
- package/src/inference/pipelines/audio/index.js +1 -20
- package/src/inference/pipelines/audio/mel.js +0 -38
- package/src/inference/pipelines/context.js +6 -2
- package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
- package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
- package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
- package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
- package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
- package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
- package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
- package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
- package/src/inference/pipelines/factory.js +14 -3
- package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +115 -124
- package/src/inference/pipelines/text/abort-contract.js +22 -0
- package/src/inference/pipelines/text/attention/attn-config.js +1 -16
- package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
- package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
- package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
- package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
- package/src/inference/pipelines/text/attention/interpreter.js +89 -439
- package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
- package/src/inference/pipelines/text/attention/plan.js +17 -0
- package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
- package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
- package/src/inference/pipelines/text/attention/projections.js +20 -537
- package/src/inference/pipelines/text/attention/query-transform.js +11 -0
- package/src/inference/pipelines/text/attention/record.js +0 -6
- package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
- package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
- package/src/inference/pipelines/text/attention/run.js +0 -6
- package/src/inference/pipelines/text/attention/types.js +3 -31
- package/src/inference/pipelines/text/chat-format.d.ts +3 -0
- package/src/inference/pipelines/text/chat-format.js +42 -0
- package/src/inference/pipelines/text/config/normalization.js +729 -0
- package/src/inference/pipelines/text/config/validation.js +9 -0
- package/src/inference/pipelines/text/config.d.ts +12 -1
- package/src/inference/pipelines/text/config.js +14 -1228
- package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
- package/src/inference/pipelines/text/debug-utils/index.js +3 -8
- package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
- package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
- package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +1 -68
- package/src/inference/pipelines/text/embed.js +178 -241
- package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
- package/src/inference/pipelines/text/embedding-contract.js +21 -0
- package/src/inference/pipelines/text/embedding-extraction.js +225 -0
- package/src/inference/pipelines/text/embedding-normalization.js +39 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
- package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
- package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
- package/src/inference/pipelines/text/execution-v1.js +7 -283
- package/src/inference/pipelines/text/execution.js +517 -0
- package/src/inference/pipelines/text/ffn/dense-executor.js +723 -0
- package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
- package/src/inference/pipelines/text/ffn/dense.js +6 -970
- package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
- package/src/inference/pipelines/text/ffn/standard.js +2 -1
- package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
- package/src/inference/pipelines/text/generator/decode-runtime.js +985 -0
- package/src/inference/pipelines/text/generator/decode.js +964 -0
- package/src/inference/pipelines/text/generator/diffusion.js +607 -0
- package/src/inference/pipelines/text/generator/logits-config.js +63 -0
- package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
- package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
- package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
- package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
- package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
- package/src/inference/pipelines/text/generator/recovery.js +310 -0
- package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
- package/src/inference/pipelines/text/generator/text.js +448 -0
- package/src/inference/pipelines/text/generator/training.js +753 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
- package/src/inference/pipelines/text/generator-logits-observation.js +48 -0
- package/src/inference/pipelines/text/generator-runtime.js +23 -154
- package/src/inference/pipelines/text/generator-steps.js +45 -980
- package/src/inference/pipelines/text/generator.js +179 -3552
- package/src/inference/pipelines/text/image-transcription.js +304 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
- package/src/inference/pipelines/text/init-rope.js +37 -189
- package/src/inference/pipelines/text/init.d.ts +8 -3
- package/src/inference/pipelines/text/init.js +6 -216
- package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
- package/src/inference/pipelines/text/layer-input-execution.js +435 -0
- package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
- package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
- package/src/inference/pipelines/text/layer-plan.js +45 -3
- package/src/inference/pipelines/text/layer.js +52 -522
- package/src/inference/pipelines/text/lifecycle.js +839 -0
- package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
- package/src/inference/pipelines/text/linear-attention.js +28 -534
- package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
- package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
- package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
- package/src/inference/pipelines/text/logits/cpu.js +8 -95
- package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
- package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
- package/src/inference/pipelines/text/logits/gpu.js +116 -579
- package/src/inference/pipelines/text/logits/index.d.ts +2 -1
- package/src/inference/pipelines/text/logits/index.js +62 -102
- package/src/inference/pipelines/text/logits/output-transform.js +54 -0
- package/src/inference/pipelines/text/logits/plan.js +184 -0
- package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
- package/src/inference/pipelines/text/logits/readback.js +9 -0
- package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/lora-apply.js +12 -4
- package/src/inference/pipelines/text/lora-types.d.ts +12 -0
- package/src/inference/pipelines/text/modality-token-contract.js +115 -0
- package/src/inference/pipelines/text/model-load.js +0 -20
- package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
- package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
- package/src/inference/pipelines/text/moe/plan.js +47 -0
- package/src/inference/pipelines/text/moe-cpu.js +9 -135
- package/src/inference/pipelines/text/moe-gpu.js +21 -545
- package/src/inference/pipelines/text/normalization-contract.js +17 -0
- package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
- package/src/inference/pipelines/text/per-layer/plan.js +362 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
- package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
- package/src/inference/pipelines/text/probes.js +20 -0
- package/src/inference/pipelines/text/sampling-config.js +22 -113
- package/src/inference/pipelines/text/sampling.d.ts +9 -1
- package/src/inference/pipelines/text/sampling.js +18 -375
- package/src/inference/pipelines/text/state.d.ts +8 -0
- package/src/inference/pipelines/text/state.js +1 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
- package/src/inference/pipelines/text/types.d.ts +7 -3
- package/src/inference/pipelines/text/weight-loading-config.js +194 -0
- package/src/inference/pipelines/text.d.ts +2 -2
- package/src/inference/pipelines/text.js +46 -1633
- package/src/inference/pipelines/video/frame-sampler.js +0 -13
- package/src/inference/pipelines/video/index.js +0 -21
- package/src/inference/pipelines/vision/attention.js +59 -0
- package/src/inference/pipelines/vision/encoder.js +254 -369
- package/src/inference/pipelines/vision/gemma4.js +204 -318
- package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
- package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
- package/src/inference/pipelines/vision/glmocr.js +557 -0
- package/src/inference/pipelines/vision/image-preprocess.js +0 -27
- package/src/inference/pipelines/vision/index.js +8 -27
- package/src/inference/pipelines/vision/ops.js +11 -33
- package/src/inference/pipelines/vision/patch-embed.js +51 -145
- package/src/inference/pipelines/vision/spatial-merge.js +78 -0
- package/src/inference/pipelines/vision/weight-loading.js +79 -0
- package/src/inference/refactor-receipt.js +2 -2
- package/src/inference/rerank.js +182 -0
- package/src/inference/token-sampling.js +319 -0
- package/src/inference/tokenizer.js +1 -1
- package/src/inference/tokenizers/behavior-flags.js +6 -3
- package/src/inference/tokenizers/bundled/decoders.js +133 -0
- package/src/inference/tokenizers/bundled/registry.js +216 -0
- package/src/inference/tokenizers/bundled.js +23 -353
- package/src/loader/doppler-loader.d.ts +4 -0
- package/src/loader/doppler-loader.js +119 -930
- package/src/loader/embedding-limit-preflight.js +11 -4
- package/src/loader/embedding-loader.js +1 -3
- package/src/loader/gpu-tensor-materialization.js +20 -0
- package/src/loader/layer-loader.js +15 -119
- package/src/loader/layer-tensor-aliases.js +99 -0
- package/src/loader/layer-weight-shape.js +29 -0
- package/src/loader/load-timing.js +64 -0
- package/src/loader/loader-types.d.ts +3 -0
- package/src/loader/model-executor.js +819 -0
- package/src/loader/model-load-validation.js +15 -0
- package/src/loader/quantization-constants.d.ts +11 -29
- package/src/loader/quantization-constants.js +1 -2
- package/src/loader/shard-resolver.js +0 -14
- package/src/loader/tensor-load-error.js +14 -0
- package/src/loader/tensor-shard-indices.js +8 -0
- package/src/loader/tensors/bf16-loader.js +125 -0
- package/src/loader/tensors/functional-descriptor-loader.js +3 -375
- package/src/loader/tensors/load-executor.js +577 -0
- package/src/loader/tensors/source-transform/executor.js +123 -0
- package/src/loader/tensors/source-transform/plan.js +107 -0
- package/src/loader/tensors/source-transform.js +15 -264
- package/src/loader/tensors/tensor-loader.d.ts +4 -4
- package/src/loader/tensors/tensor-loader.js +7 -653
- package/src/memory/buffer-pool.js +7 -29
- package/src/rules/capability-transform-resolver.js +80 -0
- package/src/rules/execution-contract.d.ts +73 -0
- package/src/rules/execution-contract.js +429 -0
- package/src/rules/execution-rules-contract-check.js +1 -1
- package/src/rules/generated/rule-bundle.json +4305 -0
- package/src/rules/inference/capability-transforms.rules.json +3 -3
- package/src/rules/layer-pattern-contract-check.js +1 -1
- package/src/rules/rule-registry.js +70 -57
- package/src/storage/artifact-storage-context.js +2 -2
- package/src/storage/backend-selection.js +43 -0
- package/src/storage/backends/opfs-store.d.ts +41 -0
- package/src/storage/distribution-transport.js +5 -0
- package/src/storage/download/integrity.d.ts +33 -0
- package/src/storage/download/integrity.js +130 -0
- package/src/storage/download/resume-inspection.d.ts +22 -0
- package/src/storage/download/resume-inspection.js +134 -0
- package/src/storage/download/retry.js +57 -0
- package/src/storage/download/state.js +131 -0
- package/src/storage/download/transport.js +99 -0
- package/src/storage/downloader.js +74 -453
- package/src/storage/model-cache.d.ts +1 -0
- package/src/storage/model-cache.js +51 -11
- package/src/storage/shard-manager.d.ts +5 -0
- package/src/storage/shard-manager.js +50 -369
- package/src/storage/shards/index.js +60 -0
- package/src/storage/shards/integrity.js +225 -0
- package/src/storage/shards/lifecycle.js +22 -0
- package/src/storage/source-artifact-store.js +14 -4
- package/src/tooling/boundary-evidence.d.ts +4 -4
- package/src/tooling/boundary-evidence.js +20 -20
- package/src/tooling/browser-command-runner.js +13 -6
- package/src/tooling/capsule-v2.d.ts +11 -0
- package/src/tooling/capsule-v2.js +58 -0
- package/src/tooling/capsule.js +20 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.js +163 -5
- package/src/tooling/command-api.d.ts +42 -3
- package/src/tooling/command-api.js +7 -2
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
- package/src/tooling/command-runner.html +49 -1
- package/src/tooling/command-validation.js +1 -24
- package/src/tooling/conversion-run-timing.js +24 -0
- package/src/tooling/diagnose-runner.js +11 -9
- package/src/tooling/distribution-shard-transport.js +11 -0
- package/src/tooling/execution-cost-ledger.d.ts +76 -2
- package/src/tooling/execution-cost-ledger.js +197 -15
- package/src/tooling/hf-registry/catalog-io.js +63 -0
- package/src/tooling/litert/package.js +863 -0
- package/src/tooling/litert-package-runtime.js +3 -849
- package/src/tooling/model-capsule-forge.js +478 -0
- package/src/tooling/node-browser/execution.d.ts +40 -0
- package/src/tooling/node-browser/execution.js +308 -0
- package/src/tooling/node-browser/static-server.js +189 -0
- package/src/tooling/node-browser/transport.d.ts +80 -0
- package/src/tooling/node-browser/transport.js +622 -0
- package/src/tooling/node-browser-command-runner.d.ts +18 -78
- package/src/tooling/node-browser-command-runner.js +10 -1112
- package/src/tooling/node-capsule-artifact-store.js +44 -0
- package/src/tooling/node-command-execution.js +237 -0
- package/src/tooling/node-command-runner.js +26 -205
- package/src/tooling/node-converter/input.js +521 -0
- package/src/tooling/node-converter/output.js +244 -0
- package/src/tooling/node-converter-chunk-batches.js +21 -0
- package/src/tooling/node-converter.js +7 -720
- package/src/tooling/node-source-runtime.js +16 -276
- package/src/tooling/node-webgpu-provider.v1.json +33 -0
- package/src/tooling/node-webgpu.js +178 -453
- package/src/tooling/opfs-cache.d.ts +5 -0
- package/src/tooling/opfs-cache.js +4 -0
- package/src/tooling/production-release.d.ts +19 -0
- package/src/tooling/production-release.js +647 -0
- package/src/tooling/program-bundle/embedding-reference.js +19 -0
- package/src/tooling/program-bundle/materialize.js +324 -0
- package/src/tooling/program-bundle/rerank-reference.js +22 -0
- package/src/tooling/program-bundle/sequence-reference.js +42 -0
- package/src/tooling/program-bundle/validation.js +31 -0
- package/src/tooling/program-bundle/wgsl-closure.js +254 -0
- package/src/tooling/program-bundle-parity.d.ts +11 -4
- package/src/tooling/program-bundle-parity.js +262 -109
- package/src/tooling/program-bundle.d.ts +7 -0
- package/src/tooling/program-bundle.js +186 -591
- package/src/tooling/provider-v1-contract.d.ts +61 -0
- package/src/tooling/provider-v1-contract.js +192 -0
- package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
- package/src/tooling/registered-variant-calibration-job.js +210 -0
- package/src/tooling/registered-variant-calibration.d.ts +167 -7
- package/src/tooling/registered-variant-calibration.js +57 -7
- package/src/tooling/runtime-optimization/candidates.js +664 -0
- package/src/tooling/runtime-optimization-campaign.js +166 -0
- package/src/tooling/runtime-optimization-index.js +16 -1
- package/src/tooling/runtime-optimization.d.ts +44 -0
- package/src/tooling/runtime-optimization.js +13 -656
- package/src/tooling/source-intake.d.ts +1 -1
- package/src/tooling/source-intake.js +34 -17
- package/src/tooling/source-package-profiles.js +1 -1
- package/src/tooling/source-runtime/bundle-materialize.js +298 -0
- package/src/tooling/source-runtime/bundle-plan.js +121 -0
- package/src/tooling/source-runtime/node-loader.js +258 -0
- package/src/tooling/source-runtime-bundle.js +26 -411
- package/src/tooling/source-runtime-converter-config.js +1 -1
- package/src/tooling-experimental-exports.js +0 -5
- package/src/tooling-exports/evidence.d.ts +14 -3
- package/src/tooling-exports/evidence.js +14 -3
- package/src/tooling-exports/storage.d.ts +7 -0
- package/src/tooling-exports/storage.js +5 -0
- package/src/tooling-exports.d.ts +3 -0
- package/src/tooling-exports.js +2 -0
- package/src/tooling-exports.shared.d.ts +17 -3
- package/src/tooling-exports.shared.js +9 -3
- package/src/training.d.ts +6 -6
- package/src/training.js +4 -4
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +512 -0
- package/tests/kernels/browser/test-page.js +170 -14
- package/tests/kernels/harness/tolerance.js +1 -0
- package/tests/kernels/reference/index.js +13 -1
- package/tests/kernels/reference/vision.js +160 -0
- package/src/converter/quantizer.d.ts +0 -121
- package/src/gpu/kernels/backward/utils.d.ts +0 -90
- package/src/gpu/kernels/uniform-utils.d.ts +0 -66
- package/src/gpu/kernels/utils.d.ts +0 -147
- package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
- package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
- package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
- package/src/inference/pipelines/text/moe-helpers.js +0 -21
- package/src/rules/converter/execution.rules.json +0 -20
- package/src/rules/converter/tensor-roles.rules.json +0 -13
- package/src/rules/converter/tokenizer.rules.json +0 -12
- package/src/rules/inference/attention.rules.json +0 -69
- package/src/rules/inference/config.rules.json +0 -82
- package/src/rules/inference/dtype.rules.json +0 -113
- package/src/rules/inference/execution.rules.json +0 -195
- package/src/rules/inference/ffn.rules.json +0 -123
- package/src/rules/inference/layer-pattern.rules.json +0 -16
- package/src/rules/inference/layer.rules.json +0 -7
- package/src/rules/inference/moe.rules.json +0 -145
- package/src/rules/kernels/attention.rules.json +0 -68
- package/src/rules/kernels/conv2d.rules.json +0 -6
- package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
- package/src/rules/kernels/dequant.rules.json +0 -58
- package/src/rules/kernels/energy.rules.json +0 -22
- package/src/rules/kernels/fused-ffn.rules.json +0 -15
- package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
- package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
- package/src/rules/kernels/gather.rules.json +0 -93
- package/src/rules/kernels/gelu.rules.json +0 -11
- package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
- package/src/rules/kernels/groupnorm.rules.json +0 -10
- package/src/rules/kernels/kv_quantize.rules.json +0 -9
- package/src/rules/kernels/layernorm.rules.json +0 -6
- package/src/rules/kernels/linear-attention.rules.json +0 -6
- package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
- package/src/rules/kernels/matmul.rules.json +0 -95
- package/src/rules/kernels/modulate.rules.json +0 -6
- package/src/rules/kernels/moe.rules.gptoss.json +0 -105
- package/src/rules/kernels/moe.rules.json +0 -14
- package/src/rules/kernels/moe.rules.mixtral.json +0 -75
- package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
- package/src/rules/kernels/relu.rules.json +0 -6
- package/src/rules/kernels/rep-penalty.rules.json +0 -6
- package/src/rules/kernels/repeat-channels.rules.json +0 -6
- package/src/rules/kernels/residual.rules.json +0 -12
- package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
- package/src/rules/kernels/rmsnorm.rules.json +0 -46
- package/src/rules/kernels/rope-qk.rules.json +0 -6
- package/src/rules/kernels/rope.rules.json +0 -6
- package/src/rules/kernels/sample.rules.json +0 -6
- package/src/rules/kernels/scale.rules.json +0 -8
- package/src/rules/kernels/silu.rules.json +0 -22
- package/src/rules/kernels/softmax.rules.json +0 -25
- package/src/rules/kernels/split-qg.rules.json +0 -6
- package/src/rules/kernels/split-qkv.rules.json +0 -6
- package/src/rules/kernels/upsample2d.rules.json +0 -6
- package/src/rules/loader/tensor-loader.rules.json +0 -42
- package/src/rules/loader/weights.rules.json +0 -42
- package/src/rules/tooling/command-runtime.rules.json +0 -65
- package/src/tooling/hf-registry-utils.js +0 -395
- /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
- /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
- /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
- /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
- /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
- /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
- /package/src/{utils → formats}/array-buffer.js +0 -0
- /package/src/{utils → formats}/canonical-hash.js +0 -0
- /package/src/{utils → formats}/clone-json.js +0 -0
- /package/src/{utils → formats}/encode-utf8.js +0 -0
- /package/src/{utils → formats}/load-json.js +0 -0
- /package/src/{utils → formats}/plain-object.js +0 -0
- /package/src/{utils → formats}/sha256.js +0 -0
- /package/src/{utils → formats}/stable-sort-object.js +0 -0
- /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
- /package/src/{utils → storage}/hf-resolve-url.js +0 -0
- /package/src/{utils → storage}/runtime-env.js +0 -0
- /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
|
@@ -28,7 +28,7 @@ import { validateCallTimeOptions } from '../../../config/param-validator.js';
|
|
|
28
28
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
29
29
|
|
|
30
30
|
// Pipeline sub-modules
|
|
31
|
-
import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './sampling.js';
|
|
31
|
+
import { sample, applyRepetitionPenalty, applyPresencePenalty, logitsSanity, getTopK } from './sampling.js';
|
|
32
32
|
import { createKVCache, isStopToken } from './init.js';
|
|
33
33
|
import { embed } from './embed.js';
|
|
34
34
|
import { processLayer } from './layer.js';
|
|
@@ -48,11 +48,13 @@ import {
|
|
|
48
48
|
import {
|
|
49
49
|
buildLayerContext,
|
|
50
50
|
debugCheckBuffer as debugCheckBufferHelper,
|
|
51
|
+
resolvePerLayerInputsSession,
|
|
52
|
+
} from './generator/session-context.js';
|
|
53
|
+
import {
|
|
51
54
|
getLogitsConfig,
|
|
52
55
|
getLogitsWeights,
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
} from './generator-helpers.js';
|
|
56
|
+
} from './generator/logits-config.js';
|
|
57
|
+
import { releaseSharedAttentionState } from './generator/attention-lifecycle.js';
|
|
56
58
|
import {
|
|
57
59
|
assertTokenIdsInRange,
|
|
58
60
|
assertTokenIdInRange,
|
|
@@ -70,7 +72,11 @@ import { resolveSamplingConfig } from './sampling-config.js';
|
|
|
70
72
|
import { decodeReadback, getLogitsHealth } from './debug-utils/index.js';
|
|
71
73
|
import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
|
|
72
74
|
import { resolveDeferredRoundingWindowTokens } from './finiteness-policy.js';
|
|
73
|
-
import {
|
|
75
|
+
import {
|
|
76
|
+
advanceDecodeStepCount,
|
|
77
|
+
createTsirFixtureState,
|
|
78
|
+
drainPendingTsirReads,
|
|
79
|
+
} from './tsir-fixture-writer.js';
|
|
74
80
|
import {
|
|
75
81
|
activateFallbackExecutionPlan,
|
|
76
82
|
hasFallbackExecutionPlan,
|
|
@@ -105,15 +111,19 @@ import {
|
|
|
105
111
|
// Extracted standalone helpers
|
|
106
112
|
import {
|
|
107
113
|
resolvePromptInput,
|
|
114
|
+
} from './generator/prompt-input.js';
|
|
115
|
+
import {
|
|
108
116
|
releasePerLayerInputBuffer,
|
|
117
|
+
shouldDisablePrefillCommandBatching,
|
|
118
|
+
resolveEffectivePrefillTokenChunkSize,
|
|
119
|
+
} from './generator/prefill-policy.js';
|
|
120
|
+
import {
|
|
109
121
|
normalizePrefixEmbeddingOverride,
|
|
110
122
|
resolvePrefillEmbeddingInputIds,
|
|
111
123
|
resolvePrefillMultimodalBidirectionalSpan,
|
|
112
124
|
applyPrefixEmbeddingOverride,
|
|
113
125
|
resolvePrefixEmbeddingOverrideTransitionDeclaredBy,
|
|
114
|
-
|
|
115
|
-
resolveEffectivePrefillTokenChunkSize,
|
|
116
|
-
} from './generator-prefill-helpers.js';
|
|
126
|
+
} from './generator/prefix-embedding.js';
|
|
117
127
|
import {
|
|
118
128
|
shouldDisableBatchDecodeAfterShortBatch,
|
|
119
129
|
resolveHotVocabularyBatchDecodeAvailability,
|
|
@@ -129,360 +139,94 @@ import {
|
|
|
129
139
|
resolveCurrentKVCacheDtype,
|
|
130
140
|
cloneRuntimeInferenceWithKVDtype,
|
|
131
141
|
} from './generator-decode-policy.js';
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
`[Pipeline] runtime.inference.session.prefillChunkSubmitMode must be "sync" or "async"; got "${String(submit)}".`
|
|
180
|
-
);
|
|
181
|
-
}
|
|
182
|
-
return submit;
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
function getTokenizerSuppressionCache(tokenizer) {
|
|
186
|
-
let cache = tokenizerSuppressionCache.get(tokenizer);
|
|
187
|
-
if (!cache) {
|
|
188
|
-
cache = new Map();
|
|
189
|
-
tokenizerSuppressionCache.set(tokenizer, cache);
|
|
190
|
-
}
|
|
191
|
-
return cache;
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
function decodeSingleTokenForSuppression(tokenizer, tokenId) {
|
|
195
|
-
if (!tokenizer || typeof tokenizer.decode !== 'function') {
|
|
196
|
-
return '';
|
|
197
|
-
}
|
|
198
|
-
try {
|
|
199
|
-
return tokenizer.decode([tokenId], false, false);
|
|
200
|
-
} catch {
|
|
201
|
-
return '';
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
function collectTokenizerSuppressedTokenIds(tokenizer, vocabSize, samplingConfig) {
|
|
206
|
-
if (!tokenizer || !Number.isInteger(vocabSize) || vocabSize < 1) {
|
|
207
|
-
return [];
|
|
208
|
-
}
|
|
209
|
-
if (samplingConfig.suppressSpecialTokens !== true && samplingConfig.suppressSpecialLikeTokens !== true) {
|
|
210
|
-
return [];
|
|
211
|
-
}
|
|
212
|
-
const cache = getTokenizerSuppressionCache(tokenizer);
|
|
213
|
-
const cacheKey = [
|
|
214
|
-
vocabSize,
|
|
215
|
-
samplingConfig.suppressSpecialTokens === true ? 'special' : 'plain',
|
|
216
|
-
samplingConfig.suppressSpecialLikeTokens === true ? 'specialLike' : 'literal',
|
|
217
|
-
].join(':');
|
|
218
|
-
const cached = cache.get(cacheKey);
|
|
219
|
-
if (cached) {
|
|
220
|
-
return cached;
|
|
221
|
-
}
|
|
222
|
-
const tokenIds = [];
|
|
223
|
-
for (let tokenId = 0; tokenId < vocabSize; tokenId++) {
|
|
224
|
-
const tokenizerSpecial = samplingConfig.suppressSpecialTokens === true
|
|
225
|
-
&& typeof tokenizer.isSpecialToken === 'function'
|
|
226
|
-
&& tokenizer.isSpecialToken(tokenId);
|
|
227
|
-
const specialLike = samplingConfig.suppressSpecialLikeTokens === true
|
|
228
|
-
&& SPECIAL_LIKE_TOKEN_RE.test(decodeSingleTokenForSuppression(tokenizer, tokenId).trim());
|
|
229
|
-
if (tokenizerSpecial || specialLike) {
|
|
230
|
-
tokenIds.push(tokenId);
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
cache.set(cacheKey, tokenIds);
|
|
234
|
-
return tokenIds;
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
function resolveSuppressedSamplingTokenIds(state, samplingConfig) {
|
|
238
|
-
const vocabSize = state.modelConfig?.vocabSize;
|
|
239
|
-
const suppressed = new Set(samplingConfig.suppressTokenIds);
|
|
240
|
-
const stopTokenIds = new Set(state.modelConfig?.stopTokenIds ?? []);
|
|
241
|
-
const eosToken = state.tokenizer?.getSpecialTokens?.()?.eos;
|
|
242
|
-
if (Number.isInteger(eosToken)) {
|
|
243
|
-
stopTokenIds.add(eosToken);
|
|
244
|
-
}
|
|
245
|
-
for (const tokenId of collectTokenizerSuppressedTokenIds(state.tokenizer, vocabSize, samplingConfig)) {
|
|
246
|
-
if (!stopTokenIds.has(tokenId)) {
|
|
247
|
-
suppressed.add(tokenId);
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
return [...suppressed];
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
function normalizeSelectedLogitTokenIds(value, vocabSize, label) {
|
|
254
|
-
if (value == null) {
|
|
255
|
-
return null;
|
|
256
|
-
}
|
|
257
|
-
if (!Array.isArray(value) && !ArrayBuffer.isView(value)) {
|
|
258
|
-
throw new Error(`${label} must be an array or typed array.`);
|
|
259
|
-
}
|
|
260
|
-
const tokenIds = Array.from(value, (entry, index) => {
|
|
261
|
-
const tokenId = Number(entry);
|
|
262
|
-
if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSize) {
|
|
263
|
-
throw new Error(`${label}[${index}] must be an integer in [0, ${vocabSize}), got "${String(entry)}".`);
|
|
264
|
-
}
|
|
265
|
-
return tokenId;
|
|
266
|
-
});
|
|
267
|
-
if (tokenIds.length === 0) {
|
|
268
|
-
throw new Error(`${label} must not be empty.`);
|
|
269
|
-
}
|
|
270
|
-
return tokenIds;
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
async function traceActivationHealth(label, buffer, dtype, elementCount) {
|
|
274
|
-
if (!isTraceEnabled('logits') || !isGpuBufferInstance(buffer)) {
|
|
275
|
-
return;
|
|
276
|
-
}
|
|
277
|
-
const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
|
|
278
|
-
const data = await readBuffer(buffer, elementCount * bytesPerElement);
|
|
279
|
-
trace.logits(label, getLogitsHealth(decodeReadback(data, dtype)));
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
function ownsBorrowedWeightBuffer(weight) {
|
|
283
|
-
return !isGpuBufferInstance(weight) && !isWeightBuffer(weight);
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
function borrowLinearWeight(weight, label) {
|
|
287
|
-
if (!weight) {
|
|
288
|
-
throw new Error(`DiffusionGemma self-conditioning missing ${label}.`);
|
|
289
|
-
}
|
|
290
|
-
if (isSplitWeightBuffer(weight)) {
|
|
291
|
-
throw new Error(
|
|
292
|
-
`DiffusionGemma self-conditioning does not support split weight storage for ${label}.`
|
|
293
|
-
);
|
|
294
|
-
}
|
|
295
|
-
return {
|
|
296
|
-
value: getPipelineWeightBuffer(weight, label),
|
|
297
|
-
owned: ownsBorrowedWeightBuffer(weight),
|
|
298
|
-
};
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
function borrowNormWeight(weight, label) {
|
|
302
|
-
if (!weight) {
|
|
303
|
-
throw new Error(`DiffusionGemma self-conditioning missing ${label}.`);
|
|
304
|
-
}
|
|
305
|
-
if (isSplitWeightBuffer(weight)) {
|
|
306
|
-
throw new Error(
|
|
307
|
-
`DiffusionGemma self-conditioning does not support split norm storage for ${label}.`
|
|
308
|
-
);
|
|
309
|
-
}
|
|
310
|
-
return {
|
|
311
|
-
value: getPipelineNormWeightBuffer(weight, label),
|
|
312
|
-
owned: ownsBorrowedWeightBuffer(weight),
|
|
313
|
-
};
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
function releaseBorrowedWeight(borrowed) {
|
|
317
|
-
if (!borrowed?.owned) {
|
|
318
|
-
return;
|
|
319
|
-
}
|
|
320
|
-
const value = borrowed.value;
|
|
321
|
-
releaseBuffer(isWeightBuffer(value) ? value.buffer : value);
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
function canUseChunkedSoftEmbeddingLogits(logitsState, embeddingWeight, embeddingTranspose) {
|
|
325
|
-
return logitsState != null
|
|
326
|
-
&& isWeightBuffer(embeddingWeight)
|
|
327
|
-
&& getWeightDtype(embeddingWeight) === 'f16'
|
|
328
|
-
&& getLayout(embeddingWeight) === 'row'
|
|
329
|
-
&& embeddingTranspose !== true;
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
function resolveDiffusionGemmaSoftEmbeddingChunkRows(runtimeConfig) {
|
|
333
|
-
const value = runtimeConfig?.inference?.diffusionGemma?.softEmbeddingLogitsChunkRows;
|
|
334
|
-
if (value == null) {
|
|
335
|
-
return undefined;
|
|
336
|
-
}
|
|
337
|
-
if (!Number.isInteger(value) || value <= 0) {
|
|
338
|
-
throw new Error(
|
|
339
|
-
'runtime.inference.diffusionGemma.softEmbeddingLogitsChunkRows must be a positive integer.'
|
|
340
|
-
);
|
|
341
|
-
}
|
|
342
|
-
return value;
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
function normalizeCanvasTokenIds(canvas, context) {
|
|
346
|
-
if (!Array.isArray(canvas) && !ArrayBuffer.isView(canvas)) {
|
|
347
|
-
throw new Error(`[DiffusionGemma] ${context}.canvas must be an array or typed array of token IDs.`);
|
|
348
|
-
}
|
|
349
|
-
return Array.from(canvas, (value, index) => {
|
|
350
|
-
if (!Number.isFinite(value) || Math.floor(value) !== value || value < 0) {
|
|
351
|
-
throw new Error(`[DiffusionGemma] ${context}.canvas[${index}] must be a non-negative integer token ID.`);
|
|
352
|
-
}
|
|
353
|
-
return value;
|
|
354
|
-
});
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
function normalizeSelfConditioningLogits(logits, canvasLength, vocabSize) {
|
|
358
|
-
if (logits == null) {
|
|
359
|
-
return null;
|
|
360
|
-
}
|
|
361
|
-
const expected = canvasLength * vocabSize;
|
|
362
|
-
const values = logits instanceof Float32Array
|
|
363
|
-
? logits
|
|
364
|
-
: (Array.isArray(logits) ? Float32Array.from(logits) : null);
|
|
365
|
-
if (!values) {
|
|
366
|
-
throw new Error('[DiffusionGemma] selfConditioningLogits must be a Float32Array or number array.');
|
|
367
|
-
}
|
|
368
|
-
if (values.length !== expected) {
|
|
369
|
-
throw new Error(
|
|
370
|
-
`[DiffusionGemma] selfConditioningLogits length mismatch: expected ${expected}, got ${values.length}.`
|
|
371
|
-
);
|
|
372
|
-
}
|
|
373
|
-
return values;
|
|
374
|
-
}
|
|
375
|
-
|
|
376
|
-
function normalizeSelfConditioningLogitsState(logits, canvasLength, vocabSize) {
|
|
377
|
-
if (logits == null || !isGpuBufferInstance(logits.logitsBuffer)) {
|
|
378
|
-
return null;
|
|
379
|
-
}
|
|
380
|
-
const dtype = logits.logitsDtype;
|
|
381
|
-
if (dtype !== 'f32') {
|
|
382
|
-
throw new Error(`[DiffusionGemma] GPU selfConditioningLogits require f32 logits, got "${dtype}".`);
|
|
383
|
-
}
|
|
384
|
-
if (logits.vocabSize !== vocabSize) {
|
|
385
|
-
throw new Error(
|
|
386
|
-
`[DiffusionGemma] GPU selfConditioningLogits vocab mismatch: expected ${vocabSize}, got ${logits.vocabSize}.`
|
|
387
|
-
);
|
|
388
|
-
}
|
|
389
|
-
if (logits.canvasLength !== canvasLength) {
|
|
390
|
-
throw new Error(
|
|
391
|
-
`[DiffusionGemma] GPU selfConditioningLogits canvas mismatch: expected ${canvasLength}, got ${logits.canvasLength}.`
|
|
392
|
-
);
|
|
393
|
-
}
|
|
394
|
-
const temperature = logits.temperature ?? 1.0;
|
|
395
|
-
if (typeof temperature !== 'number' || !Number.isFinite(temperature) || temperature <= 0) {
|
|
396
|
-
throw new Error('[DiffusionGemma] GPU selfConditioningLogits temperature must be positive.');
|
|
397
|
-
}
|
|
398
|
-
return {
|
|
399
|
-
logitsBuffer: logits.logitsBuffer,
|
|
400
|
-
logitsDtype: dtype,
|
|
401
|
-
temperature,
|
|
402
|
-
releaseOnUse: logits.releaseOnUse !== false,
|
|
403
|
-
};
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
function normalizeSelfConditioningSoftEmbeddingState(state, canvasLength, hiddenSize) {
|
|
407
|
-
if (state?.kind !== 'soft_embedding') {
|
|
408
|
-
return null;
|
|
409
|
-
}
|
|
410
|
-
if (!isGpuBufferInstance(state.buffer)) {
|
|
411
|
-
throw new Error('[DiffusionGemma] GPU selfConditioning soft embedding requires a GPU buffer.');
|
|
412
|
-
}
|
|
413
|
-
const dtype = state.dtype ?? 'f32';
|
|
414
|
-
if (dtype !== 'f32') {
|
|
415
|
-
throw new Error(`[DiffusionGemma] GPU selfConditioning soft embedding requires f32 dtype, got "${dtype}".`);
|
|
416
|
-
}
|
|
417
|
-
if (state.canvasLength !== canvasLength) {
|
|
418
|
-
throw new Error(
|
|
419
|
-
`[DiffusionGemma] GPU selfConditioning soft embedding canvas mismatch: ` +
|
|
420
|
-
`expected ${canvasLength}, got ${state.canvasLength}.`
|
|
421
|
-
);
|
|
422
|
-
}
|
|
423
|
-
if (state.hiddenSize !== hiddenSize) {
|
|
424
|
-
throw new Error(
|
|
425
|
-
`[DiffusionGemma] GPU selfConditioning soft embedding hidden mismatch: ` +
|
|
426
|
-
`expected ${hiddenSize}, got ${state.hiddenSize}.`
|
|
427
|
-
);
|
|
428
|
-
}
|
|
429
|
-
return {
|
|
430
|
-
buffer: state.buffer,
|
|
431
|
-
dtype,
|
|
432
|
-
releaseOnUse: state.releaseOnUse !== false,
|
|
433
|
-
scaled: state.scaled === true,
|
|
434
|
-
};
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
let intentBundleModulePromise = null;
|
|
438
|
-
|
|
439
|
-
async function getExperimentalIntentBundleModule() {
|
|
440
|
-
intentBundleModulePromise ??= import('../../../experimental/hotswap/intent-bundle.js');
|
|
441
|
-
return intentBundleModulePromise;
|
|
442
|
-
}
|
|
443
|
-
|
|
142
|
+
import { FINITENESS_RESET_WORDS, borrowLinearWeight, borrowNormWeight, canUseChunkedSoftEmbeddingLogits, getExperimentalIntentBundleModule, normalizeCanvasTokenIds, normalizeSelectedLogitTokenIds, normalizeSelfConditioningLogits, normalizeSelfConditioningLogitsState, normalizeSelfConditioningSoftEmbeddingState, recordPrefillRecorderStats, releaseBorrowedWeight, resolveDiffusionGemmaSoftEmbeddingChunkRows, resolvePrefillChunkSubmitMode, resolveSuppressedSamplingTokenIds, traceActivationHealth } from './generator/text.js';
|
|
143
|
+
import {
|
|
144
|
+
_createDiffusionGemmaSelfConditioningEmbeddings as _createDiffusionGemmaSelfConditioningEmbeddingsImpl,
|
|
145
|
+
_createDiffusionGemmaSelfConditioningSoftEmbeddingState as _createDiffusionGemmaSelfConditioningSoftEmbeddingStateImpl,
|
|
146
|
+
computeDiffusionGemmaCanvasLogits as computeDiffusionGemmaCanvasLogitsImpl,
|
|
147
|
+
computeDiffusionGemmaCanvasStep as computeDiffusionGemmaCanvasStepImpl,
|
|
148
|
+
} from './generator/diffusion.js';
|
|
149
|
+
import {
|
|
150
|
+
_replayPrefillDecodeLogits as _replayPrefillDecodeLogitsImpl,
|
|
151
|
+
_shouldUseFinitenessFallback as _shouldUseFinitenessFallbackImpl,
|
|
152
|
+
_recreateKVCacheForExecutionPlan as _recreateKVCacheForExecutionPlanImpl,
|
|
153
|
+
_openFinitenessFallbackWindow as _openFinitenessFallbackWindowImpl,
|
|
154
|
+
_closeFinitenessFallbackWindow as _closeFinitenessFallbackWindowImpl,
|
|
155
|
+
_consumeFinitenessFallbackToken as _consumeFinitenessFallbackTokenImpl,
|
|
156
|
+
_beginFinitenessFallback as _beginFinitenessFallbackImpl,
|
|
157
|
+
_endFinitenessFallback as _endFinitenessFallbackImpl,
|
|
158
|
+
_retryWithFinitenessFallback as _retryWithFinitenessFallbackImpl,
|
|
159
|
+
_retryWithPersistentFinitenessFallback as _retryWithPersistentFinitenessFallbackImpl,
|
|
160
|
+
_retryDecodeStepWithFinitenessWindow as _retryDecodeStepWithFinitenessWindowImpl,
|
|
161
|
+
} from './generator/recovery.js';
|
|
162
|
+
import {
|
|
163
|
+
prefillKVOnly as prefillKVOnlyImpl,
|
|
164
|
+
prefillForLoRATraining as prefillForLoRATrainingImpl,
|
|
165
|
+
prefillWithEmbedding as prefillWithEmbeddingImpl,
|
|
166
|
+
prefillWithLogits as prefillWithLogitsImpl,
|
|
167
|
+
prefillWithTokenLogits as prefillWithTokenLogitsImpl,
|
|
168
|
+
prefillWithTokenLogitsFromKV as prefillWithTokenLogitsFromKVImpl,
|
|
169
|
+
generateWithPrefixKV as generateWithPrefixKVImpl,
|
|
170
|
+
} from './generator/training.js';
|
|
171
|
+
import {
|
|
172
|
+
_generateTokensInternal as _generateTokensInternalImpl,
|
|
173
|
+
generateTokenIds as generateTokenIdsImpl,
|
|
174
|
+
_runDecodeLoop as _runDecodeLoopImpl,
|
|
175
|
+
_decodeStep as _decodeStepImpl,
|
|
176
|
+
decodeStepLogits as decodeStepLogitsImpl,
|
|
177
|
+
advanceWithToken as advanceWithTokenImpl,
|
|
178
|
+
advanceWithTokenAndEmbedding as advanceWithTokenAndEmbeddingImpl,
|
|
179
|
+
_generateNTokensGPU as _generateNTokensGPUImpl,
|
|
180
|
+
} from './generator/decode-runtime.js';
|
|
181
|
+
import {
|
|
182
|
+
_commitPrefillHiddenChunk as _commitPrefillHiddenChunkImpl,
|
|
183
|
+
_prefillInputIdsToLogits as _prefillInputIdsToLogitsImpl,
|
|
184
|
+
_prefillPromptToLogits as _prefillPromptToLogitsImpl,
|
|
185
|
+
_prefillToHidden as _prefillToHiddenImpl,
|
|
186
|
+
_prefill as _prefillImpl,
|
|
187
|
+
} from './generator/prefill-runtime.js';
|
|
188
|
+
export { resolvePrefillChunkSubmitMode } from './generator/text.js';
|
|
444
189
|
|
|
445
190
|
export class PipelineGenerator {
|
|
446
191
|
|
|
447
|
-
|
|
448
|
-
|
|
192
|
+
_state;
|
|
193
|
+
_finitenessFallbackWindow;
|
|
449
194
|
|
|
450
195
|
_assertTokenIdsInRange(tokenIds, context = 'encode') {
|
|
451
|
-
assertTokenIdsInRange(this
|
|
196
|
+
assertTokenIdsInRange(this._state, tokenIds, context);
|
|
452
197
|
}
|
|
453
198
|
|
|
454
199
|
_assertTokenIdInRange(tokenId, context = 'token') {
|
|
455
|
-
assertTokenIdInRange(this
|
|
200
|
+
assertTokenIdInRange(this._state, tokenId, context);
|
|
456
201
|
}
|
|
457
202
|
|
|
458
|
-
|
|
459
203
|
constructor(state) {
|
|
460
|
-
this
|
|
461
|
-
this
|
|
204
|
+
this._state = state;
|
|
205
|
+
this._finitenessFallbackWindow = null;
|
|
462
206
|
}
|
|
463
207
|
|
|
464
208
|
_resolveDeferredRoundingWindowTokens() {
|
|
465
|
-
const activePlan = resolveActiveExecutionPlan(this
|
|
209
|
+
const activePlan = resolveActiveExecutionPlan(this._state);
|
|
466
210
|
return activePlan?.deferredRoundingWindowTokens
|
|
467
|
-
?? resolveDeferredRoundingWindowTokens(this
|
|
211
|
+
?? resolveDeferredRoundingWindowTokens(this._state.runtimeConfig?.inference?.compute);
|
|
468
212
|
}
|
|
469
213
|
|
|
470
214
|
_getEffectiveActivationDtype() {
|
|
471
|
-
return resolveActiveExecutionPlan(this
|
|
215
|
+
return resolveActiveExecutionPlan(this._state).activationDtype;
|
|
472
216
|
}
|
|
473
217
|
|
|
474
218
|
_hasFinitenessFallbackWindow() {
|
|
475
|
-
return this
|
|
219
|
+
return this._finitenessFallbackWindow !== null;
|
|
476
220
|
}
|
|
477
221
|
|
|
478
222
|
_resetReplayPrefillRuntimeState() {
|
|
479
|
-
this
|
|
480
|
-
this
|
|
481
|
-
this
|
|
223
|
+
this._state.kvCache?.clear?.();
|
|
224
|
+
this._state.linearAttentionRuntime = resetLinearAttentionRuntime(this._state.linearAttentionRuntime);
|
|
225
|
+
this._state.currentSeqLen = 0;
|
|
482
226
|
}
|
|
483
227
|
|
|
484
228
|
resetGenerationState() {
|
|
485
|
-
if (this
|
|
229
|
+
if (this._state.isGenerating) {
|
|
486
230
|
throw new Error('InferencePipeline.resetGenerationState: cannot reset while generation is in progress');
|
|
487
231
|
}
|
|
488
232
|
this._resetReplayPrefillRuntimeState();
|
|
@@ -490,120 +234,58 @@ export class PipelineGenerator {
|
|
|
490
234
|
}
|
|
491
235
|
|
|
492
236
|
async _replayPrefillDecodeLogits(currentIds, opts) {
|
|
493
|
-
|
|
494
|
-
// Without KV cache creation, this bound is not enforced elsewhere.
|
|
495
|
-
const kvConfig = this.#state.runtimeConfig?.inference?.session?.kvcache;
|
|
496
|
-
const replayMaxSeqLen = kvConfig?.maxSeqLen;
|
|
497
|
-
if (Number.isFinite(replayMaxSeqLen) && replayMaxSeqLen > 0 && currentIds.length > replayMaxSeqLen) {
|
|
498
|
-
throw new Error(
|
|
499
|
-
`[Pipeline] Replay-prefill sequence length ${currentIds.length} exceeds ` +
|
|
500
|
-
`runtime.inference.session.kvcache.maxSeqLen (${replayMaxSeqLen}). ` +
|
|
501
|
-
'Increase maxSeqLen in a tier profile or runtime config to allow longer sequences.'
|
|
502
|
-
);
|
|
503
|
-
}
|
|
504
|
-
this.#state.decodeStepCount++;
|
|
505
|
-
this._resetReplayPrefillRuntimeState();
|
|
506
|
-
const logits = await this._prefill(currentIds, opts);
|
|
507
|
-
return {
|
|
508
|
-
logits,
|
|
509
|
-
logitsBuffer: null,
|
|
510
|
-
logitsDtype: null,
|
|
511
|
-
rawVocabSize: this.#state.modelConfig.vocabSize,
|
|
512
|
-
vocabSize: this.#state.modelConfig.vocabSize,
|
|
513
|
-
};
|
|
237
|
+
return _replayPrefillDecodeLogitsImpl.apply(this, arguments);
|
|
514
238
|
}
|
|
515
239
|
|
|
516
240
|
_shouldUseFinitenessFallback(error, contextLabel) {
|
|
517
|
-
|
|
518
|
-
return false;
|
|
519
|
-
}
|
|
520
|
-
if (!hasFallbackExecutionPlan(this.#state)) {
|
|
521
|
-
throw createUnhandledFinitenessPolicyError(this.#state, contextLabel, error);
|
|
522
|
-
}
|
|
523
|
-
return true;
|
|
241
|
+
return _shouldUseFinitenessFallbackImpl.apply(this, arguments);
|
|
524
242
|
}
|
|
525
243
|
|
|
526
244
|
_recreateKVCacheForExecutionPlan(plan, reasonLabel) {
|
|
527
|
-
|
|
528
|
-
const runtimeInference = cloneRuntimeInferenceWithKVDtype(this.#state, kvDtype);
|
|
529
|
-
this.#state.kvCache?.destroy?.();
|
|
530
|
-
this.#state.kvCache = createKVCache(
|
|
531
|
-
this.#state.modelConfig,
|
|
532
|
-
this.#state.useGPU,
|
|
533
|
-
this.#state.debug,
|
|
534
|
-
runtimeInference
|
|
535
|
-
);
|
|
536
|
-
this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
|
|
537
|
-
this.#state.currentSeqLen = 0;
|
|
538
|
-
return kvDtype;
|
|
245
|
+
return _recreateKVCacheForExecutionPlanImpl.apply(this, arguments);
|
|
539
246
|
}
|
|
540
247
|
|
|
541
248
|
_openFinitenessFallbackWindow(opts, reasonLabel, tokenCount, rollbackSeqLen = undefined) {
|
|
542
|
-
|
|
543
|
-
? Math.max(1, Math.floor(tokenCount))
|
|
544
|
-
: 1;
|
|
545
|
-
if (this.#finitenessFallbackWindow) {
|
|
546
|
-
this.#finitenessFallbackWindow.remainingTokens = Math.max(
|
|
547
|
-
this.#finitenessFallbackWindow.remainingTokens,
|
|
548
|
-
normalizedCount
|
|
549
|
-
);
|
|
550
|
-
return;
|
|
551
|
-
}
|
|
552
|
-
const original = this._beginFinitenessFallback(opts, reasonLabel, rollbackSeqLen);
|
|
553
|
-
this.#finitenessFallbackWindow = {
|
|
554
|
-
original,
|
|
555
|
-
remainingTokens: normalizedCount,
|
|
556
|
-
};
|
|
249
|
+
return _openFinitenessFallbackWindowImpl.apply(this, arguments);
|
|
557
250
|
}
|
|
558
251
|
|
|
559
252
|
_closeFinitenessFallbackWindow(opts) {
|
|
560
|
-
|
|
561
|
-
return;
|
|
562
|
-
}
|
|
563
|
-
const original = this.#finitenessFallbackWindow.original;
|
|
564
|
-
this.#finitenessFallbackWindow = null;
|
|
565
|
-
this._endFinitenessFallback(opts, original);
|
|
253
|
+
return _closeFinitenessFallbackWindowImpl.apply(this, arguments);
|
|
566
254
|
}
|
|
567
255
|
|
|
568
256
|
_consumeFinitenessFallbackToken(opts) {
|
|
569
|
-
|
|
570
|
-
return;
|
|
571
|
-
}
|
|
572
|
-
this.#finitenessFallbackWindow.remainingTokens -= 1;
|
|
573
|
-
if (this.#finitenessFallbackWindow.remainingTokens <= 0) {
|
|
574
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
575
|
-
}
|
|
257
|
+
return _consumeFinitenessFallbackTokenImpl.apply(this, arguments);
|
|
576
258
|
}
|
|
577
259
|
|
|
578
260
|
_resolveStepOptions(options = {}) {
|
|
579
|
-
return resolveStepOptions(this
|
|
261
|
+
return resolveStepOptions(this._state, options);
|
|
580
262
|
}
|
|
581
263
|
|
|
582
264
|
_resetDecodeRuntimeState() {
|
|
583
|
-
this
|
|
584
|
-
this
|
|
585
|
-
this
|
|
586
|
-
this
|
|
587
|
-
this
|
|
588
|
-
this
|
|
589
|
-
this
|
|
590
|
-
this
|
|
591
|
-
this
|
|
592
|
-
this
|
|
593
|
-
this
|
|
594
|
-
this
|
|
595
|
-
this
|
|
596
|
-
this
|
|
597
|
-
this
|
|
598
|
-
this
|
|
599
|
-
this
|
|
600
|
-
this
|
|
601
|
-
this
|
|
602
|
-
this
|
|
603
|
-
this
|
|
604
|
-
this
|
|
605
|
-
this
|
|
606
|
-
this
|
|
265
|
+
this._state.stats.prefillProfileSteps = [];
|
|
266
|
+
this._state.stats.decodeMode = null;
|
|
267
|
+
this._state.stats.batchGuardReason = null;
|
|
268
|
+
this._state.stats.decodeProfileSteps = [];
|
|
269
|
+
this._state.stats.ttftMs = 0;
|
|
270
|
+
this._state.stats.decodeTimeMs = 0;
|
|
271
|
+
this._state.stats.prefillRecordMs = 0;
|
|
272
|
+
this._state.stats.prefillRecordOps = 0;
|
|
273
|
+
this._state.stats.prefillRecordPasses = 0;
|
|
274
|
+
this._state.stats.prefillRecordOpLabels = {};
|
|
275
|
+
this._state.stats.prefillSubmitWaitMs = 0;
|
|
276
|
+
this._state.stats.decodeRecordMs = 0;
|
|
277
|
+
this._state.stats.decodeRecordOps = 0;
|
|
278
|
+
this._state.stats.decodeRecordPasses = 0;
|
|
279
|
+
this._state.stats.decodeRecordOpLabels = {};
|
|
280
|
+
this._state.stats.decodeSubmitWaitMs = 0;
|
|
281
|
+
this._state.stats.decodeReadbackWaitMs = 0;
|
|
282
|
+
this._state.stats.decodeReadbackMapWaitMs = 0;
|
|
283
|
+
this._state.stats.decodeReadbackCleanupMs = 0;
|
|
284
|
+
this._state.stats.decodeReadbackCopyMs = 0;
|
|
285
|
+
this._state.decodeStepCount = 0;
|
|
286
|
+
this._state.disableRecordedLogits = false;
|
|
287
|
+
this._state.disableFusedDecode = false;
|
|
288
|
+
this._state.batchingStats = {
|
|
607
289
|
batchedForwardCalls: 0,
|
|
608
290
|
unbatchedForwardCalls: 0,
|
|
609
291
|
totalBatchedTimeMs: 0,
|
|
@@ -616,23 +298,23 @@ export class PipelineGenerator {
|
|
|
616
298
|
maxBatchTokenCap: null,
|
|
617
299
|
batchClampCount: 0,
|
|
618
300
|
};
|
|
619
|
-
resetActiveExecutionPlan(this
|
|
620
|
-
this
|
|
301
|
+
resetActiveExecutionPlan(this._state);
|
|
302
|
+
this._state.decodeRing?.reset();
|
|
621
303
|
}
|
|
622
304
|
|
|
623
305
|
_getDecodeHelpers(debugCheckBuffer) {
|
|
624
306
|
return {
|
|
625
307
|
buildLayerContext: (recorder, isDecodeMode, debugLayers, executionPlan) =>
|
|
626
|
-
buildLayerContext(this
|
|
627
|
-
getLogitsWeights: () => getLogitsWeights(this
|
|
628
|
-
getLogitsConfig: () => getLogitsConfig(this
|
|
308
|
+
buildLayerContext(this._state, recorder, isDecodeMode, debugLayers, debugCheckBuffer, executionPlan),
|
|
309
|
+
getLogitsWeights: () => getLogitsWeights(this._state),
|
|
310
|
+
getLogitsConfig: () => getLogitsConfig(this._state),
|
|
629
311
|
releaseSharedAttentionState,
|
|
630
312
|
debugCheckBuffer,
|
|
631
313
|
};
|
|
632
314
|
}
|
|
633
315
|
|
|
634
316
|
async _getFinalNormWeights() {
|
|
635
|
-
return getFinalNormWeights(this
|
|
317
|
+
return getFinalNormWeights(this._state);
|
|
636
318
|
}
|
|
637
319
|
|
|
638
320
|
_extractEmbeddingFromHidden(hiddenStates, numTokens, hiddenSize, embeddingMode, finalNormWeights, config) {
|
|
@@ -643,9 +325,9 @@ export class PipelineGenerator {
|
|
|
643
325
|
embeddingMode,
|
|
644
326
|
finalNormWeights,
|
|
645
327
|
config,
|
|
646
|
-
this
|
|
328
|
+
this._state.embeddingPostprocessor,
|
|
647
329
|
null,
|
|
648
|
-
this
|
|
330
|
+
this._state.weights.get('final_norm_bias') ?? null
|
|
649
331
|
);
|
|
650
332
|
}
|
|
651
333
|
|
|
@@ -656,13 +338,13 @@ export class PipelineGenerator {
|
|
|
656
338
|
hiddenSize,
|
|
657
339
|
finalNormWeights,
|
|
658
340
|
config,
|
|
659
|
-
this
|
|
341
|
+
this._state.weights.get('final_norm_bias') ?? null
|
|
660
342
|
);
|
|
661
343
|
}
|
|
662
344
|
|
|
663
345
|
_resolvePromptTokenIds(prompt, useChatTemplate, contextLabel) {
|
|
664
|
-
const processedPrompt = resolvePromptInput(this
|
|
665
|
-
const inputIds = this
|
|
346
|
+
const processedPrompt = resolvePromptInput(this._state, prompt, useChatTemplate, contextLabel);
|
|
347
|
+
const inputIds = this._state.tokenizer.encode(processedPrompt);
|
|
666
348
|
this._assertTokenIdsInRange(inputIds, `${contextLabel}.encode`);
|
|
667
349
|
return inputIds;
|
|
668
350
|
}
|
|
@@ -677,23 +359,29 @@ export class PipelineGenerator {
|
|
|
677
359
|
|
|
678
360
|
_sampleNextTokenFromLogits(logits, generatedIds, opts) {
|
|
679
361
|
const sampledLogits = Float32Array.from(logits);
|
|
680
|
-
applyRepetitionPenalty(sampledLogits, generatedIds, opts.repetitionPenalty);
|
|
362
|
+
applyRepetitionPenalty(sampledLogits, generatedIds, opts.repetitionPenalty, opts.repetitionPenaltyWindow);
|
|
363
|
+
if (opts.presencePenalty) {
|
|
364
|
+
applyPresencePenalty(sampledLogits, generatedIds, opts.presencePenalty, opts.repetitionPenaltyWindow);
|
|
365
|
+
}
|
|
681
366
|
// Optional pre-sample logit mask. Callers pass `opts.logitMaskFn` to
|
|
682
367
|
// implement grammar/schema-constrained decoding. The hook receives the
|
|
683
368
|
// mutable logit buffer (after repetition penalty) plus the running token
|
|
684
369
|
// sequence so it can track parse state across decode steps.
|
|
685
370
|
if (typeof opts?.logitMaskFn === "function") {
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
371
|
+
if (!Number.isSafeInteger(opts.promptTokenCount) || opts.promptTokenCount < 0 || opts.promptTokenCount > generatedIds.length) {
|
|
372
|
+
throw new Error('[Pipeline] logitMaskFn requires an explicit prompt token boundary.');
|
|
373
|
+
}
|
|
374
|
+
const result = opts.logitMaskFn(sampledLogits, {
|
|
375
|
+
generatedIds: generatedIds.slice(opts.promptTokenCount),
|
|
376
|
+
tokenizer: this._state.tokenizer ?? null,
|
|
377
|
+
vocabSize: this._state.modelConfig?.vocabSize ?? sampledLogits.length,
|
|
378
|
+
});
|
|
379
|
+
if (result && typeof result.then === 'function') {
|
|
380
|
+
Promise.resolve(result).catch(() => {});
|
|
381
|
+
throw new Error('[Pipeline] logitMaskFn must be synchronous.');
|
|
694
382
|
}
|
|
695
383
|
}
|
|
696
|
-
const padTokenId = this
|
|
384
|
+
const padTokenId = this._state.tokenizer?.getSpecialTokens?.()?.pad;
|
|
697
385
|
const tokenId = sample(sampledLogits, {
|
|
698
386
|
temperature: opts.temperature,
|
|
699
387
|
topP: opts.topP,
|
|
@@ -712,7 +400,7 @@ export class PipelineGenerator {
|
|
|
712
400
|
}
|
|
713
401
|
|
|
714
402
|
_resolvePrefillTokenChunkSize(inputIds) {
|
|
715
|
-
const chunkSize = resolveEffectivePrefillTokenChunkSize(this
|
|
403
|
+
const chunkSize = resolveEffectivePrefillTokenChunkSize(this._state);
|
|
716
404
|
if (chunkSize === undefined) {
|
|
717
405
|
throw new Error('inference.session.prefillTokenChunkSize is required; use null to disable token-chunked prefill.');
|
|
718
406
|
}
|
|
@@ -726,105 +414,26 @@ export class PipelineGenerator {
|
|
|
726
414
|
}
|
|
727
415
|
|
|
728
416
|
async _commitPrefillHiddenChunk(prefillResult) {
|
|
729
|
-
|
|
730
|
-
numTokens,
|
|
731
|
-
startPos,
|
|
732
|
-
currentRecorder,
|
|
733
|
-
recordProfile,
|
|
734
|
-
currentHiddenBuffer,
|
|
735
|
-
} = prefillResult;
|
|
736
|
-
|
|
737
|
-
try {
|
|
738
|
-
if (currentRecorder) {
|
|
739
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
740
|
-
await currentRecorder.submitAndWait();
|
|
741
|
-
await recordProfile(currentRecorder);
|
|
742
|
-
} else {
|
|
743
|
-
const device = getDevice();
|
|
744
|
-
if (device) {
|
|
745
|
-
await device.queue.onSubmittedWorkDone();
|
|
746
|
-
}
|
|
747
|
-
}
|
|
748
|
-
this.#state.currentSeqLen = startPos + numTokens;
|
|
749
|
-
} finally {
|
|
750
|
-
releaseBuffer(currentHiddenBuffer);
|
|
751
|
-
}
|
|
417
|
+
return _commitPrefillHiddenChunkImpl.apply(this, arguments);
|
|
752
418
|
}
|
|
753
419
|
|
|
754
420
|
async _prefillInputIdsToLogits(inputIds, opts) {
|
|
755
|
-
|
|
756
|
-
if (chunkSize === null) {
|
|
757
|
-
return this._prefill(inputIds, opts);
|
|
758
|
-
}
|
|
759
|
-
|
|
760
|
-
for (let offset = 0; offset < inputIds.length; offset += chunkSize) {
|
|
761
|
-
const end = Math.min(offset + chunkSize, inputIds.length);
|
|
762
|
-
const chunk = inputIds.slice(offset, end);
|
|
763
|
-
const isFinalChunk = end === inputIds.length;
|
|
764
|
-
if (isFinalChunk) {
|
|
765
|
-
return this._prefill(chunk, opts);
|
|
766
|
-
}
|
|
767
|
-
const prefillResult = await this._prefillToHidden(chunk, opts);
|
|
768
|
-
await this._commitPrefillHiddenChunk(prefillResult);
|
|
769
|
-
}
|
|
770
|
-
|
|
771
|
-
return this._prefill(inputIds, opts);
|
|
421
|
+
return _prefillInputIdsToLogitsImpl.apply(this, arguments);
|
|
772
422
|
}
|
|
773
423
|
|
|
774
424
|
async _prefillPromptToLogits(prompt, opts, contextLabel) {
|
|
775
|
-
|
|
776
|
-
const inputStart = performance.now();
|
|
777
|
-
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, contextLabel, opts.inputIds);
|
|
778
|
-
const inputMs = performance.now() - inputStart;
|
|
779
|
-
if (opts.debug) {
|
|
780
|
-
log.debug('Pipeline', `${contextLabel}: ${inputIds.length} tokens`);
|
|
781
|
-
}
|
|
782
|
-
|
|
783
|
-
let logits;
|
|
784
|
-
const prefillStart = performance.now();
|
|
785
|
-
const runPrefill = () => this._prefillInputIdsToLogits(inputIds, opts);
|
|
786
|
-
try {
|
|
787
|
-
logits = await runPrefill();
|
|
788
|
-
} catch (error) {
|
|
789
|
-
if (!this._shouldUseFinitenessFallback(error, contextLabel)) {
|
|
790
|
-
throw error;
|
|
791
|
-
}
|
|
792
|
-
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during ${contextLabel}. Retrying with F32 precision.`);
|
|
793
|
-
logits = await this._retryWithPersistentFinitenessFallback(
|
|
794
|
-
opts,
|
|
795
|
-
contextLabel,
|
|
796
|
-
opts.maxTokens ?? 1,
|
|
797
|
-
runPrefill,
|
|
798
|
-
prefillStartSeqLen
|
|
799
|
-
);
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
return {
|
|
803
|
-
inputIds,
|
|
804
|
-
logits,
|
|
805
|
-
phase: {
|
|
806
|
-
inputMs,
|
|
807
|
-
prefillMs: performance.now() - prefillStart,
|
|
808
|
-
tokens: inputIds.length,
|
|
809
|
-
prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
|
|
810
|
-
prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
|
|
811
|
-
prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
|
|
812
|
-
prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
|
|
813
|
-
prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
|
|
814
|
-
gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
|
|
815
|
-
},
|
|
816
|
-
};
|
|
425
|
+
return _prefillPromptToLogitsImpl.apply(this, arguments);
|
|
817
426
|
}
|
|
818
427
|
|
|
819
428
|
async _decodeStepToLogits(currentIds, opts) {
|
|
820
|
-
if (usesReplayPrefillDecode(this
|
|
429
|
+
if (usesReplayPrefillDecode(this._state)) {
|
|
821
430
|
return this._replayPrefillDecodeLogits(currentIds, opts);
|
|
822
431
|
}
|
|
823
|
-
const debugCheckBuffer = this
|
|
432
|
+
const debugCheckBuffer = this._state.debug
|
|
824
433
|
? (buffer, label, numTokens, expectedDim) =>
|
|
825
|
-
debugCheckBufferHelper(this
|
|
434
|
+
debugCheckBufferHelper(this._state, buffer, label, numTokens, expectedDim)
|
|
826
435
|
: undefined;
|
|
827
|
-
return decodeStepLogits(this
|
|
436
|
+
return decodeStepLogits(this._state, currentIds, opts, this._getDecodeHelpers(debugCheckBuffer));
|
|
828
437
|
}
|
|
829
438
|
|
|
830
439
|
async _decodeNextTokenViaLogits(currentIds, opts) {
|
|
@@ -836,7 +445,7 @@ export class PipelineGenerator {
|
|
|
836
445
|
if (!Array.isArray(stopSequences) || stopSequences.length === 0) {
|
|
837
446
|
return false;
|
|
838
447
|
}
|
|
839
|
-
const fullText = this
|
|
448
|
+
const fullText = this._state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
840
449
|
return stopSequences.some((sequence) => fullText.endsWith(sequence));
|
|
841
450
|
}
|
|
842
451
|
|
|
@@ -848,429 +457,32 @@ export class PipelineGenerator {
|
|
|
848
457
|
}
|
|
849
458
|
|
|
850
459
|
_recordStopReason(reason, tokenId = null) {
|
|
851
|
-
this
|
|
852
|
-
this
|
|
460
|
+
this._state.stats.stopReason = reason;
|
|
461
|
+
this._state.stats.stopTokenId = Number.isInteger(tokenId) ? tokenId : null;
|
|
853
462
|
}
|
|
854
463
|
|
|
855
464
|
async *_generateTokensInternal(prompt, options = {}, mode = 'text') {
|
|
856
|
-
|
|
857
|
-
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
858
|
-
|
|
859
|
-
validateCallTimeOptions(options);
|
|
860
|
-
|
|
861
|
-
this.#state.isGenerating = true;
|
|
862
|
-
this._resetDecodeRuntimeState();
|
|
863
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
864
|
-
this.#state.stats.gpuTimeDecodeMs = undefined;
|
|
865
|
-
this.#state.stats.decodeRecordMs = 0;
|
|
866
|
-
this.#state.stats.decodeRecordOps = 0;
|
|
867
|
-
this.#state.stats.decodeRecordPasses = 0;
|
|
868
|
-
this.#state.stats.decodeRecordOpLabels = {};
|
|
869
|
-
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
870
|
-
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
871
|
-
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
872
|
-
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
873
|
-
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
874
|
-
this.#state.stats.prefillRecordMs = 0;
|
|
875
|
-
this.#state.stats.prefillRecordOps = 0;
|
|
876
|
-
this.#state.stats.prefillRecordPasses = 0;
|
|
877
|
-
this.#state.stats.prefillRecordOpLabels = {};
|
|
878
|
-
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
879
|
-
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
880
|
-
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
881
|
-
this.#state.stats.singleTokenReadbackMapWaitMs = 0;
|
|
882
|
-
this.#state.stats.singleTokenReadbackCleanupMs = 0;
|
|
883
|
-
this.#state.stats.singleTokenReadbackCopyMs = 0;
|
|
884
|
-
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
885
|
-
this.#state.stats.pleHotVocabularyHits = 0;
|
|
886
|
-
this.#state.stats.pleHotVocabularyMisses = 0;
|
|
887
|
-
this.#state.stats.plePreparedTokenCacheHits = 0;
|
|
888
|
-
this.#state.stats.plePreparedTokenCacheMisses = 0;
|
|
889
|
-
this.#state.stats.plePreparedTokenCacheEntries = 0;
|
|
890
|
-
this.#state.stats.plePreparedTokenCacheBytes = 0;
|
|
891
|
-
this.#state.stats.decodeMode = null;
|
|
892
|
-
this.#state.stats.batchGuardReason = null;
|
|
893
|
-
this.#state.stats.stopReason = null;
|
|
894
|
-
this.#state.stats.stopTokenId = null;
|
|
895
|
-
this.#state.stats.ttftMs = 0;
|
|
896
|
-
const startTime = performance.now();
|
|
897
|
-
|
|
898
|
-
const opts = resolveGenerateOptions(this.#state, options);
|
|
899
|
-
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
900
|
-
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
901
|
-
// Validate and normalize sampling parameters through single source of truth
|
|
902
|
-
const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
|
|
903
|
-
opts.temperature = samplingConfig.temperature;
|
|
904
|
-
opts.topP = samplingConfig.topP;
|
|
905
|
-
opts.topK = samplingConfig.topK;
|
|
906
|
-
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
907
|
-
opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this.#state, samplingConfig);
|
|
908
|
-
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
909
|
-
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
910
|
-
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
911
|
-
if (diagnosticsEnabled || tsirFixtureCfg) {
|
|
912
|
-
const captureConfig = {
|
|
913
|
-
...createDefaultCaptureConfig(),
|
|
914
|
-
enabled: true,
|
|
915
|
-
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
916
|
-
...(options?.diagnostics?.captureConfig ?? {}),
|
|
917
|
-
};
|
|
918
|
-
validateCaptureConfig(captureConfig);
|
|
919
|
-
this.#state.operatorDiagnostics = {
|
|
920
|
-
enabled: diagnosticsEnabled === true,
|
|
921
|
-
captureConfig,
|
|
922
|
-
emitter: diagnosticsEnabled ? new OperatorEventEmitter({
|
|
923
|
-
modelHash: this.#state.manifest?.modelId ?? null,
|
|
924
|
-
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
925
|
-
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
926
|
-
}) : null,
|
|
927
|
-
tsirFixture: tsirFixtureCfg ? {
|
|
928
|
-
dir: tsirFixtureCfg.dir,
|
|
929
|
-
layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
|
|
930
|
-
? tsirFixtureCfg.layerFilter
|
|
931
|
-
: null,
|
|
932
|
-
records: [],
|
|
933
|
-
} : null,
|
|
934
|
-
};
|
|
935
|
-
}
|
|
936
|
-
const activePlan = opts.executionPlan ?? resolveActiveExecutionPlan(this.#state);
|
|
937
|
-
this.#state.stats.executionPlan = {
|
|
938
|
-
primary: summarizeExecutionPlan(this.#state.executionPlanState?.primaryPlan ?? null),
|
|
939
|
-
fallback: summarizeExecutionPlan(this.#state.executionPlanState?.fallbackPlan ?? null),
|
|
940
|
-
activePlanIdAtStart: activePlan?.id ?? null,
|
|
941
|
-
finalActivePlanId: activePlan?.id ?? null,
|
|
942
|
-
transitions: [],
|
|
943
|
-
};
|
|
944
|
-
this.#state.stats.kernelPathId = activePlan?.kernelPathId ?? this.#state.resolvedKernelPath?.id ?? null;
|
|
945
|
-
this.#state.stats.kernelPathSource = activePlan?.kernelPathSource ?? this.#state.kernelPathSource ?? 'none';
|
|
946
|
-
|
|
947
|
-
if (opts.debug) {
|
|
948
|
-
log.debug('Pipeline', `ChatTemplate: options=${options.useChatTemplate}, final=${opts.useChatTemplate}`);
|
|
949
|
-
}
|
|
950
|
-
|
|
951
|
-
const emitToken = async function* (generator, tokenId, textDecoder) {
|
|
952
|
-
if (mode === 'token') {
|
|
953
|
-
yield tokenId;
|
|
954
|
-
if (options.onToken) options.onToken(tokenId, '');
|
|
955
|
-
return;
|
|
956
|
-
}
|
|
957
|
-
const tokenText = textDecoder(tokenId);
|
|
958
|
-
yield tokenText;
|
|
959
|
-
if (options.onToken) options.onToken(tokenId, tokenText);
|
|
960
|
-
};
|
|
961
|
-
|
|
962
|
-
try {
|
|
963
|
-
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
964
|
-
const prefillStart = performance.now();
|
|
965
|
-
const { inputIds, logits: initialPrefillLogits } = await this._prefillPromptToLogits(prompt, opts, 'generate');
|
|
966
|
-
let prefillLogits = initialPrefillLogits;
|
|
967
|
-
this.#state.stats.prefillTimeMs = performance.now() - prefillStart;
|
|
968
|
-
this._assertTokenIdsInRange(inputIds, 'generate.prefillTokens');
|
|
969
|
-
const generatedIds = [...inputIds];
|
|
970
|
-
this.#state.stats.prefillTokens = inputIds.length;
|
|
971
|
-
|
|
972
|
-
if (opts.debug) {
|
|
973
|
-
log.debug('Pipeline', `Input: ${inputIds.length} tokens`);
|
|
974
|
-
}
|
|
975
|
-
|
|
976
|
-
const intentBundleConfig = this.#state.runtimeConfig.shared.intentBundle;
|
|
977
|
-
const intentBundle = intentBundleConfig?.bundle;
|
|
978
|
-
const expectedTopK = intentBundle?.payload?.expectedTopK
|
|
979
|
-
?? intentBundle?.payload?.expected_top_k;
|
|
980
|
-
const maxDriftThreshold = intentBundle?.constraints?.maxDriftThreshold
|
|
981
|
-
?? intentBundle?.constraints?.max_drift_threshold;
|
|
982
|
-
|
|
983
|
-
if (intentBundleConfig?.enabled && Array.isArray(expectedTopK) && expectedTopK.length > 0) {
|
|
984
|
-
const { enforceLogitDrift } = await getExperimentalIntentBundleModule();
|
|
985
|
-
const actualTopK = getTopK(
|
|
986
|
-
prefillLogits,
|
|
987
|
-
expectedTopK.length,
|
|
988
|
-
(tokens) => resolveTokenText(this.#state.tokenizer, tokens),
|
|
989
|
-
).map((token) => token.token);
|
|
990
|
-
const driftResult = enforceLogitDrift(expectedTopK, actualTopK, maxDriftThreshold);
|
|
991
|
-
if (!driftResult.ok) {
|
|
992
|
-
throw new Error(`Intent bundle drift check failed: ${driftResult.reason}`);
|
|
993
|
-
}
|
|
994
|
-
}
|
|
995
|
-
|
|
996
|
-
if (opts.debug) {
|
|
997
|
-
const topAfterPenalty = getTopK(
|
|
998
|
-
Float32Array.from(prefillLogits),
|
|
999
|
-
5,
|
|
1000
|
-
(tokens) => resolveTokenText(this.#state.tokenizer, tokens)
|
|
1001
|
-
);
|
|
1002
|
-
log.debug('Pipeline', `After rep penalty top-5: ${topAfterPenalty.map(t => `"${t.text}"(${(t.prob * 100).toFixed(1)}%)`).join(', ')}`);
|
|
1003
|
-
}
|
|
1004
|
-
|
|
1005
|
-
let firstToken;
|
|
1006
|
-
try {
|
|
1007
|
-
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
1008
|
-
} catch (error) {
|
|
1009
|
-
if (!this._shouldUseFinitenessFallback(error, 'prefill-sample')) {
|
|
1010
|
-
throw error;
|
|
1011
|
-
}
|
|
1012
|
-
log.warn('Pipeline', 'FinitenessGuard caught non-finite prefill logits at sampling. Retrying with F32 precision.');
|
|
1013
|
-
prefillLogits = await this._retryWithPersistentFinitenessFallback(
|
|
1014
|
-
opts,
|
|
1015
|
-
'prefill-sample',
|
|
1016
|
-
opts.maxTokens,
|
|
1017
|
-
() => this._prefill(inputIds, opts),
|
|
1018
|
-
prefillStartSeqLen
|
|
1019
|
-
);
|
|
1020
|
-
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
1021
|
-
}
|
|
1022
|
-
|
|
1023
|
-
if (opts.debug) {
|
|
1024
|
-
const firstTokenText = resolveTokenText(this.#state.tokenizer, [firstToken], `[${firstToken}]`, (tokens) => this.#state.tokenizer?.decode?.(tokens, true, false));
|
|
1025
|
-
log.debug('Pipeline', `First token sampled: id=${firstToken} text="${firstTokenText}"`);
|
|
1026
|
-
}
|
|
1027
|
-
|
|
1028
|
-
const stopTokenIds = this.#state.modelConfig.stopTokenIds;
|
|
1029
|
-
const eosToken = this.#state.tokenizer.getSpecialTokens?.()?.eos;
|
|
1030
|
-
const stopSequenceStart = inputIds.length;
|
|
1031
|
-
generatedIds.push(firstToken);
|
|
1032
|
-
this.#state.stats.ttftMs = performance.now() - startTime;
|
|
1033
|
-
|
|
1034
|
-
const decodeToken = (tokenId) => resolveTokenText(
|
|
1035
|
-
this.#state.tokenizer,
|
|
1036
|
-
[tokenId],
|
|
1037
|
-
`[${tokenId}]`,
|
|
1038
|
-
(tokens) => this.#state.tokenizer?.decode?.(tokens, true, false),
|
|
1039
|
-
(tokens) => this.#state.tokenizer?.decode?.(tokens, false, false)
|
|
1040
|
-
);
|
|
1041
|
-
const decodeRuntime = {
|
|
1042
|
-
stopTokenIds,
|
|
1043
|
-
eosToken,
|
|
1044
|
-
stopSequenceStart,
|
|
1045
|
-
decodeToken,
|
|
1046
|
-
logBatchPath: opts.debug,
|
|
1047
|
-
emitMode: mode,
|
|
1048
|
-
};
|
|
1049
|
-
|
|
1050
|
-
yield* emitToken(this, firstToken, decodeToken);
|
|
1051
|
-
|
|
1052
|
-
if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
|
|
1053
|
-
this._recordStopReason('stop-token-or-sequence', firstToken);
|
|
1054
|
-
this.#state.stats.decodeTimeMs = 0;
|
|
1055
|
-
this.#state.stats.tokensGenerated = 1;
|
|
1056
|
-
this.#state.stats.decodeTokens = 1;
|
|
1057
|
-
} else {
|
|
1058
|
-
yield* this._runDecodeLoop(generatedIds, opts, options, decodeRuntime);
|
|
1059
|
-
}
|
|
1060
|
-
const tokensGenerated = this.#state.stats.decodeTokens ?? 1;
|
|
1061
|
-
this.#state.stats.totalTimeMs = performance.now() - startTime;
|
|
1062
|
-
|
|
1063
|
-
if (opts.debug) {
|
|
1064
|
-
log.debug('Pipeline', `Generated ${tokensGenerated} tokens in ${this.#state.stats.totalTimeMs.toFixed(0)}ms`);
|
|
1065
|
-
}
|
|
1066
|
-
|
|
1067
|
-
const ttft = this.#state.stats.ttftMs ?? this.#state.stats.prefillTimeMs;
|
|
1068
|
-
const decodeTokens = Math.max(0, tokensGenerated - 1);
|
|
1069
|
-
const decodeSpeed = decodeTokens > 0 ? (decodeTokens / this.#state.stats.decodeTimeMs * 1000) : 0;
|
|
1070
|
-
const loadMs = this.#state.stats.modelLoadMs;
|
|
1071
|
-
const loadLabel = Number.isFinite(loadMs) ? `Load: ${loadMs.toFixed(0)}ms | ` : '';
|
|
1072
|
-
if (opts.benchmark) {
|
|
1073
|
-
log.info('Benchmark', `${loadLabel}TTFT: ${ttft.toFixed(0)}ms | Prefill: ${this.#state.stats.prefillTimeMs.toFixed(0)}ms | Decode: ${this.#state.stats.decodeTimeMs.toFixed(0)}ms (${decodeTokens} tokens @ ${decodeSpeed.toFixed(1)} tok/s)`);
|
|
1074
|
-
} else {
|
|
1075
|
-
log.info('Perf', `${loadLabel}TTFT: ${ttft.toFixed(0)}ms | Prefill: ${this.#state.stats.prefillTimeMs.toFixed(0)}ms | Decode: ${this.#state.stats.decodeTimeMs.toFixed(0)}ms (${decodeTokens} tokens @ ${decodeSpeed.toFixed(1)} tok/s)`);
|
|
1076
|
-
}
|
|
1077
|
-
trace.perf('Decode summary', {
|
|
1078
|
-
ttftMs: ttft,
|
|
1079
|
-
prefillMs: this.#state.stats.prefillTimeMs,
|
|
1080
|
-
decodeMs: this.#state.stats.decodeTimeMs,
|
|
1081
|
-
decodeTokens,
|
|
1082
|
-
decodeSpeed,
|
|
1083
|
-
totalMs: this.#state.stats.totalTimeMs,
|
|
1084
|
-
});
|
|
1085
|
-
} finally {
|
|
1086
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
1087
|
-
resetActiveExecutionPlan(this.#state);
|
|
1088
|
-
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
1089
|
-
? {
|
|
1090
|
-
enabled: true,
|
|
1091
|
-
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
1092
|
-
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
1093
|
-
}
|
|
1094
|
-
: null;
|
|
1095
|
-
this.#state.operatorDiagnostics = null;
|
|
1096
|
-
this.#state.isGenerating = false;
|
|
1097
|
-
}
|
|
465
|
+
return yield* _generateTokensInternalImpl.apply(this, arguments);
|
|
1098
466
|
}
|
|
1099
467
|
|
|
1100
468
|
_beginFinitenessFallback(opts, reasonLabel, rollbackSeqLen = undefined) {
|
|
1101
|
-
|
|
1102
|
-
const currentKvDtype = resolveCurrentKVCacheDtype(
|
|
1103
|
-
this.#state,
|
|
1104
|
-
originalPlan,
|
|
1105
|
-
`${reasonLabel}: current plan`
|
|
1106
|
-
);
|
|
1107
|
-
const original = {
|
|
1108
|
-
activePlanId: this.#state.executionPlanState?.activePlanId ?? 'primary',
|
|
1109
|
-
seed: opts.seed,
|
|
1110
|
-
restoreKVCachePlan: null,
|
|
1111
|
-
};
|
|
1112
|
-
|
|
1113
|
-
const fallbackPlan = activateFallbackExecutionPlan(this.#state);
|
|
1114
|
-
if (!fallbackPlan) {
|
|
1115
|
-
throw new Error(
|
|
1116
|
-
'[Pipeline] Explicit alternate-plan finiteness recovery is unavailable for this model/runtime configuration.'
|
|
1117
|
-
);
|
|
1118
|
-
}
|
|
1119
|
-
log.warn(
|
|
1120
|
-
'Pipeline',
|
|
1121
|
-
`FinitenessGuard fallback (${reasonLabel}): ` +
|
|
1122
|
-
`${originalPlan.kernelPathId ?? 'none'} -> ${fallbackPlan.kernelPathId ?? 'none'}`
|
|
1123
|
-
);
|
|
1124
|
-
const fallbackKvDtype = resolveTargetPlanKVDtype(
|
|
1125
|
-
fallbackPlan,
|
|
1126
|
-
`${reasonLabel}: fallback plan`
|
|
1127
|
-
);
|
|
1128
|
-
|
|
1129
|
-
if (Number.isInteger(rollbackSeqLen) && rollbackSeqLen < 0) {
|
|
1130
|
-
setActiveExecutionPlan(this.#state, original.activePlanId);
|
|
1131
|
-
throw new Error(
|
|
1132
|
-
`[Pipeline] ${reasonLabel}: rollbackSeqLen must be a non-negative integer when provided.`
|
|
1133
|
-
);
|
|
1134
|
-
}
|
|
1135
|
-
|
|
1136
|
-
if (fallbackKvDtype !== currentKvDtype) {
|
|
1137
|
-
if (rollbackSeqLen !== 0) {
|
|
1138
|
-
setActiveExecutionPlan(this.#state, original.activePlanId);
|
|
1139
|
-
throw new Error(
|
|
1140
|
-
`[Pipeline] ${reasonLabel}: finiteness fallback requires rebuilding the KV cache ` +
|
|
1141
|
-
`${currentKvDtype} -> ${fallbackKvDtype}, which is only supported from a fresh prefill (rollbackSeqLen=0).`
|
|
1142
|
-
);
|
|
1143
|
-
}
|
|
1144
|
-
try {
|
|
1145
|
-
this._recreateKVCacheForExecutionPlan(fallbackPlan, reasonLabel);
|
|
1146
|
-
original.restoreKVCachePlan = originalPlan;
|
|
1147
|
-
} catch (error) {
|
|
1148
|
-
setActiveExecutionPlan(this.#state, original.activePlanId);
|
|
1149
|
-
try {
|
|
1150
|
-
this._recreateKVCacheForExecutionPlan(originalPlan, `${reasonLabel}: restore primary`);
|
|
1151
|
-
} catch (restoreError) {
|
|
1152
|
-
log.warn(
|
|
1153
|
-
'Pipeline',
|
|
1154
|
-
`Failed to restore primary KV cache after fallback activation error: ${restoreError}`
|
|
1155
|
-
);
|
|
1156
|
-
}
|
|
1157
|
-
throw error;
|
|
1158
|
-
}
|
|
1159
|
-
} else if (Number.isInteger(rollbackSeqLen)) {
|
|
1160
|
-
this.#state.kvCache?.truncate(rollbackSeqLen);
|
|
1161
|
-
this.#state.currentSeqLen = rollbackSeqLen;
|
|
1162
|
-
if (rollbackSeqLen === 0) {
|
|
1163
|
-
this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
|
|
1164
|
-
}
|
|
1165
|
-
} else {
|
|
1166
|
-
this.#state.kvCache?.truncate(this.#state.currentSeqLen);
|
|
1167
|
-
}
|
|
1168
|
-
|
|
1169
|
-
this.#state.decodeBuffers?.ensureBuffers({
|
|
1170
|
-
hiddenSize: this.#state.modelConfig.hiddenSize,
|
|
1171
|
-
intermediateSize: this.#state.modelConfig.maxIntermediateSize,
|
|
1172
|
-
activationDtype: fallbackPlan.activationDtype,
|
|
1173
|
-
enablePingPong: true,
|
|
1174
|
-
});
|
|
1175
|
-
|
|
1176
|
-
if (opts.seed == null) {
|
|
1177
|
-
const fallbackSeedBase = (this.#state.decodeStepCount + this.#state.currentSeqLen + 1) >>> 0;
|
|
1178
|
-
opts.seed = (fallbackSeedBase * 2654435761) >>> 0;
|
|
1179
|
-
}
|
|
1180
|
-
opts.executionPlan = rebaseExecutionSessionPlan(this.#state, opts.executionPlan);
|
|
1181
|
-
if (this.#state.stats.executionPlan) {
|
|
1182
|
-
this.#state.stats.executionPlan.finalActivePlanId = fallbackPlan.id;
|
|
1183
|
-
this.#state.stats.executionPlan.transitions.push({
|
|
1184
|
-
kind: 'activate-finiteness-fallback',
|
|
1185
|
-
reason: reasonLabel ?? null,
|
|
1186
|
-
decodeStep: this.#state.decodeStepCount,
|
|
1187
|
-
seqLen: this.#state.currentSeqLen,
|
|
1188
|
-
fromPlanId: originalPlan.id,
|
|
1189
|
-
toPlanId: fallbackPlan.id,
|
|
1190
|
-
fromKernelPathId: originalPlan.kernelPathId ?? null,
|
|
1191
|
-
toKernelPathId: fallbackPlan.kernelPathId ?? null,
|
|
1192
|
-
});
|
|
1193
|
-
}
|
|
1194
|
-
this.#state.stats.kernelPathId = fallbackPlan.kernelPathId ?? null;
|
|
1195
|
-
this.#state.stats.kernelPathSource = fallbackPlan.kernelPathSource ?? 'none';
|
|
1196
|
-
|
|
1197
|
-
return original;
|
|
469
|
+
return _beginFinitenessFallbackImpl.apply(this, arguments);
|
|
1198
470
|
}
|
|
1199
471
|
|
|
1200
472
|
_endFinitenessFallback(opts, original) {
|
|
1201
|
-
|
|
1202
|
-
setActiveExecutionPlan(this.#state, original.activePlanId);
|
|
1203
|
-
opts.executionPlan = rebaseExecutionSessionPlan(this.#state, opts.executionPlan);
|
|
1204
|
-
const restoredPlan = resolveActiveExecutionPlan(this.#state);
|
|
1205
|
-
if (original.restoreKVCachePlan) {
|
|
1206
|
-
this._recreateKVCacheForExecutionPlan(restoredPlan, 'restore-primary-plan');
|
|
1207
|
-
}
|
|
1208
|
-
if (this.#state.stats.executionPlan) {
|
|
1209
|
-
this.#state.stats.executionPlan.finalActivePlanId = restoredPlan.id;
|
|
1210
|
-
this.#state.stats.executionPlan.transitions.push({
|
|
1211
|
-
kind: 'restore-primary-plan',
|
|
1212
|
-
reason: null,
|
|
1213
|
-
decodeStep: this.#state.decodeStepCount,
|
|
1214
|
-
seqLen: this.#state.currentSeqLen,
|
|
1215
|
-
fromPlanId: this.#state.executionPlanState?.fallbackPlan?.id ?? null,
|
|
1216
|
-
toPlanId: restoredPlan.id,
|
|
1217
|
-
fromKernelPathId: this.#state.executionPlanState?.fallbackPlan?.kernelPathId ?? null,
|
|
1218
|
-
toKernelPathId: restoredPlan.kernelPathId ?? null,
|
|
1219
|
-
});
|
|
1220
|
-
}
|
|
1221
|
-
this.#state.stats.kernelPathId = restoredPlan.kernelPathId ?? this.#state.resolvedKernelPath?.id ?? null;
|
|
1222
|
-
this.#state.stats.kernelPathSource = restoredPlan.kernelPathSource ?? this.#state.kernelPathSource ?? 'none';
|
|
1223
|
-
const nextActivationDtype = this._getEffectiveActivationDtype();
|
|
1224
|
-
this.#state.decodeBuffers?.ensureBuffers({
|
|
1225
|
-
hiddenSize: this.#state.modelConfig.hiddenSize,
|
|
1226
|
-
intermediateSize: this.#state.modelConfig.maxIntermediateSize,
|
|
1227
|
-
activationDtype: nextActivationDtype,
|
|
1228
|
-
enablePingPong: true,
|
|
1229
|
-
});
|
|
473
|
+
return _endFinitenessFallbackImpl.apply(this, arguments);
|
|
1230
474
|
}
|
|
1231
475
|
|
|
1232
476
|
async _retryWithFinitenessFallback(opts, reasonLabel, retryFn, rollbackSeqLen = undefined) {
|
|
1233
|
-
|
|
1234
|
-
return retryFn();
|
|
1235
|
-
}
|
|
1236
|
-
const original = this._beginFinitenessFallback(opts, reasonLabel, rollbackSeqLen);
|
|
1237
|
-
try {
|
|
1238
|
-
return await retryFn();
|
|
1239
|
-
} finally {
|
|
1240
|
-
this._endFinitenessFallback(opts, original);
|
|
1241
|
-
}
|
|
477
|
+
return _retryWithFinitenessFallbackImpl.apply(this, arguments);
|
|
1242
478
|
}
|
|
1243
479
|
|
|
1244
480
|
async _retryWithPersistentFinitenessFallback(opts, reasonLabel, tokenBudget, retryFn, rollbackSeqLen = undefined) {
|
|
1245
|
-
|
|
1246
|
-
return retryFn();
|
|
1247
|
-
}
|
|
1248
|
-
this._openFinitenessFallbackWindow(opts, reasonLabel, tokenBudget, rollbackSeqLen);
|
|
1249
|
-
try {
|
|
1250
|
-
return await retryFn();
|
|
1251
|
-
} catch (error) {
|
|
1252
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
1253
|
-
throw error;
|
|
1254
|
-
}
|
|
481
|
+
return _retryWithPersistentFinitenessFallbackImpl.apply(this, arguments);
|
|
1255
482
|
}
|
|
1256
483
|
|
|
1257
484
|
async _retryDecodeStepWithFinitenessWindow(generatedIds, opts, reasonLabel) {
|
|
1258
|
-
|
|
1259
|
-
if (windowTokens <= 1) {
|
|
1260
|
-
return this._retryWithFinitenessFallback(
|
|
1261
|
-
opts,
|
|
1262
|
-
reasonLabel,
|
|
1263
|
-
() => this._decodeStep(generatedIds, opts)
|
|
1264
|
-
);
|
|
1265
|
-
}
|
|
1266
|
-
|
|
1267
|
-
this._openFinitenessFallbackWindow(opts, reasonLabel, windowTokens);
|
|
1268
|
-
try {
|
|
1269
|
-
return await this._decodeStep(generatedIds, opts);
|
|
1270
|
-
} catch (error) {
|
|
1271
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
1272
|
-
throw error;
|
|
1273
|
-
}
|
|
485
|
+
return _retryDecodeStepWithFinitenessWindowImpl.apply(this, arguments);
|
|
1274
486
|
}
|
|
1275
487
|
|
|
1276
488
|
// ==========================================================================
|
|
@@ -1287,591 +499,36 @@ export class PipelineGenerator {
|
|
|
1287
499
|
* Only valid when no decode is in progress.
|
|
1288
500
|
*/
|
|
1289
501
|
resetToSeqLen(seqLen) {
|
|
1290
|
-
if (this
|
|
502
|
+
if (this._state.isGenerating) {
|
|
1291
503
|
throw new Error('InferencePipeline.resetToSeqLen: cannot reset while generation is in progress');
|
|
1292
504
|
}
|
|
1293
505
|
const target = Math.max(0, Math.floor(Number(seqLen) || 0));
|
|
1294
506
|
if (!Number.isFinite(target)) {
|
|
1295
507
|
throw new Error('InferencePipeline.resetToSeqLen: seqLen must be a finite non-negative integer');
|
|
1296
508
|
}
|
|
1297
|
-
if (target > this
|
|
509
|
+
if (target > this._state.currentSeqLen) {
|
|
1298
510
|
throw new Error(
|
|
1299
|
-
`InferencePipeline.resetToSeqLen: target ${target} exceeds currentSeqLen ${this
|
|
511
|
+
`InferencePipeline.resetToSeqLen: target ${target} exceeds currentSeqLen ${this._state.currentSeqLen}`
|
|
1300
512
|
);
|
|
1301
513
|
}
|
|
1302
|
-
this
|
|
1303
|
-
this
|
|
514
|
+
this._state.kvCache?.truncate?.(target);
|
|
515
|
+
this._state.currentSeqLen = target;
|
|
1304
516
|
}
|
|
1305
517
|
|
|
1306
518
|
async _createDiffusionGemmaSelfConditioningEmbeddings(canvasIds, selfConditioningLogits, opts) {
|
|
1307
|
-
|
|
1308
|
-
const canvasLength = canvasIds.length;
|
|
1309
|
-
const hiddenSize = config.hiddenSize;
|
|
1310
|
-
const vocabSize = config.vocabSize;
|
|
1311
|
-
const elementCount = canvasLength * hiddenSize;
|
|
1312
|
-
const softEmbeddingState = normalizeSelfConditioningSoftEmbeddingState(
|
|
1313
|
-
selfConditioningLogits,
|
|
1314
|
-
canvasLength,
|
|
1315
|
-
hiddenSize
|
|
1316
|
-
);
|
|
1317
|
-
const logitsState = softEmbeddingState
|
|
1318
|
-
? null
|
|
1319
|
-
: normalizeSelfConditioningLogitsState(selfConditioningLogits, canvasLength, vocabSize);
|
|
1320
|
-
const logits = (softEmbeddingState || logitsState)
|
|
1321
|
-
? null
|
|
1322
|
-
: normalizeSelfConditioningLogits(selfConditioningLogits, canvasLength, vocabSize);
|
|
1323
|
-
const weights = this.#state.weights.get('diffusion_gemma_self_conditioning');
|
|
1324
|
-
if (!weights || typeof weights !== 'object') {
|
|
1325
|
-
throw new Error(
|
|
1326
|
-
'DiffusionGemma self-conditioning weights were not loaded. ' +
|
|
1327
|
-
'Expected model.decoder.self_conditioning tensors in the manifest.'
|
|
1328
|
-
);
|
|
1329
|
-
}
|
|
1330
|
-
|
|
1331
|
-
const embedBufferRaw = this.#state.weights.get('embed');
|
|
1332
|
-
if (!embedBufferRaw) {
|
|
1333
|
-
throw new Error('DiffusionGemma self-conditioning requires loaded embed_tokens weights.');
|
|
1334
|
-
}
|
|
1335
|
-
const borrowed = {
|
|
1336
|
-
preNorm: null,
|
|
1337
|
-
gateProj: null,
|
|
1338
|
-
upProj: null,
|
|
1339
|
-
downProj: null,
|
|
1340
|
-
softEmbedding: null,
|
|
1341
|
-
};
|
|
1342
|
-
let softLogitsTensor = null;
|
|
1343
|
-
let softLogitsOwned = true;
|
|
1344
|
-
let softEmbeddingStateTensor = null;
|
|
1345
|
-
let softEmbeddingStateOwned = false;
|
|
1346
|
-
let softmaxTensor = null;
|
|
1347
|
-
let softEmbeddings = null;
|
|
1348
|
-
let scaledSoftEmbeddings = null;
|
|
1349
|
-
let baseEmbeddings = null;
|
|
1350
|
-
let normedSoft = null;
|
|
1351
|
-
let gate = null;
|
|
1352
|
-
let up = null;
|
|
1353
|
-
let activated = null;
|
|
1354
|
-
let down = null;
|
|
1355
|
-
let combined = null;
|
|
1356
|
-
let output = null;
|
|
1357
|
-
|
|
1358
|
-
const releaseTensorOnce = (() => {
|
|
1359
|
-
const released = new Set();
|
|
1360
|
-
return (tensor) => {
|
|
1361
|
-
const buffer = tensor?.buffer ?? null;
|
|
1362
|
-
if (!buffer || released.has(buffer)) return;
|
|
1363
|
-
released.add(buffer);
|
|
1364
|
-
releaseBuffer(buffer);
|
|
1365
|
-
};
|
|
1366
|
-
})();
|
|
1367
|
-
|
|
1368
|
-
try {
|
|
1369
|
-
borrowed.preNorm = borrowNormWeight(weights.preNorm, 'diffusion_gemma_self_conditioning.pre_norm');
|
|
1370
|
-
borrowed.gateProj = borrowLinearWeight(weights.gateProj, 'diffusion_gemma_self_conditioning.gate_proj');
|
|
1371
|
-
borrowed.upProj = borrowLinearWeight(weights.upProj, 'diffusion_gemma_self_conditioning.up_proj');
|
|
1372
|
-
borrowed.downProj = borrowLinearWeight(weights.downProj, 'diffusion_gemma_self_conditioning.down_proj');
|
|
1373
|
-
if (isSplitWeightBuffer(embedBufferRaw) && this.#state.embeddingTranspose === true) {
|
|
1374
|
-
throw new Error(
|
|
1375
|
-
'DiffusionGemma self-conditioning split embeddings require row-major embedding storage.'
|
|
1376
|
-
);
|
|
1377
|
-
}
|
|
1378
|
-
borrowed.softEmbedding = isSplitWeightBuffer(embedBufferRaw)
|
|
1379
|
-
? null
|
|
1380
|
-
: borrowLinearWeight(embedBufferRaw, 'diffusion_gemma_self_conditioning.embed_tokens');
|
|
1381
|
-
|
|
1382
|
-
const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
|
|
1383
|
-
const embedDtype = isCpuWeightBuffer(embedBufferRaw)
|
|
1384
|
-
? embedBufferRaw.dtype
|
|
1385
|
-
: getWeightDtype(embedBufferRaw);
|
|
1386
|
-
const embedMetadata = getWeightMetadata(embedBufferRaw);
|
|
1387
|
-
baseEmbeddings = await embed(canvasIds, embedBuffer, {
|
|
1388
|
-
hiddenSize,
|
|
1389
|
-
vocabSize,
|
|
1390
|
-
scaleEmbeddings: config.scaleEmbeddings,
|
|
1391
|
-
embeddingScale: config.embeddingScale,
|
|
1392
|
-
debug: opts.debug,
|
|
1393
|
-
recorder: null,
|
|
1394
|
-
transpose: this.#state.embeddingTranspose,
|
|
1395
|
-
activationDtype: 'f32',
|
|
1396
|
-
embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
|
|
1397
|
-
embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
|
|
1398
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1399
|
-
operatorDiagnostics: this.#state.operatorDiagnostics,
|
|
1400
|
-
});
|
|
1401
|
-
|
|
1402
|
-
if (softEmbeddingState) {
|
|
1403
|
-
softEmbeddingStateOwned = softEmbeddingState.releaseOnUse;
|
|
1404
|
-
softEmbeddings = createTensor(
|
|
1405
|
-
softEmbeddingState.buffer,
|
|
1406
|
-
softEmbeddingState.dtype,
|
|
1407
|
-
[canvasLength, hiddenSize],
|
|
1408
|
-
'diffusion_gemma_self_conditioning_soft_embedding'
|
|
1409
|
-
);
|
|
1410
|
-
softEmbeddingStateTensor = softEmbeddings;
|
|
1411
|
-
if (!softEmbeddingState.scaled) {
|
|
1412
|
-
scaledSoftEmbeddings = await runScale(softEmbeddings, Math.sqrt(hiddenSize), {
|
|
1413
|
-
count: elementCount,
|
|
1414
|
-
});
|
|
1415
|
-
if (softEmbeddingStateOwned) {
|
|
1416
|
-
releaseTensorOnce(softEmbeddings);
|
|
1417
|
-
}
|
|
1418
|
-
softEmbeddings = scaledSoftEmbeddings;
|
|
1419
|
-
softEmbeddingStateTensor = null;
|
|
1420
|
-
scaledSoftEmbeddings = null;
|
|
1421
|
-
}
|
|
1422
|
-
} else if (logitsState || logits) {
|
|
1423
|
-
const softmaxTemperature = logitsState?.temperature ?? 1.0;
|
|
1424
|
-
if (logitsState) {
|
|
1425
|
-
softLogitsOwned = logitsState.releaseOnUse;
|
|
1426
|
-
softLogitsTensor = createTensor(
|
|
1427
|
-
logitsState.logitsBuffer,
|
|
1428
|
-
logitsState.logitsDtype,
|
|
1429
|
-
[canvasLength, vocabSize],
|
|
1430
|
-
'diffusion_gemma_self_conditioning_logits'
|
|
1431
|
-
);
|
|
1432
|
-
} else {
|
|
1433
|
-
const logitsBuffer = acquireBuffer(logits.byteLength, undefined, 'diffusion_gemma_self_conditioning_logits');
|
|
1434
|
-
uploadData(logitsBuffer, logits);
|
|
1435
|
-
softLogitsTensor = createTensor(logitsBuffer, 'f32', [canvasLength, vocabSize], 'diffusion_gemma_self_conditioning_logits');
|
|
1436
|
-
}
|
|
1437
|
-
if (canUseChunkedSoftEmbeddingLogits(
|
|
1438
|
-
logitsState,
|
|
1439
|
-
borrowed.softEmbedding?.value,
|
|
1440
|
-
this.#state.embeddingTranspose
|
|
1441
|
-
)) {
|
|
1442
|
-
softEmbeddings = await runSoftEmbeddingLogitsF16(
|
|
1443
|
-
softLogitsTensor,
|
|
1444
|
-
borrowed.softEmbedding.value,
|
|
1445
|
-
canvasLength,
|
|
1446
|
-
hiddenSize,
|
|
1447
|
-
vocabSize,
|
|
1448
|
-
{
|
|
1449
|
-
temperature: softmaxTemperature,
|
|
1450
|
-
chunkRows: resolveDiffusionGemmaSoftEmbeddingChunkRows(this.#state.runtimeConfig),
|
|
1451
|
-
}
|
|
1452
|
-
);
|
|
1453
|
-
} else {
|
|
1454
|
-
softmaxTensor = await runSoftmax(softLogitsTensor, -1, {
|
|
1455
|
-
batchSize: canvasLength,
|
|
1456
|
-
size: vocabSize,
|
|
1457
|
-
temperature: softmaxTemperature,
|
|
1458
|
-
});
|
|
1459
|
-
softEmbeddings = isSplitWeightBuffer(embedBufferRaw)
|
|
1460
|
-
? await runSoftEmbeddingSplitF16(
|
|
1461
|
-
softmaxTensor,
|
|
1462
|
-
embedBufferRaw,
|
|
1463
|
-
canvasLength,
|
|
1464
|
-
hiddenSize,
|
|
1465
|
-
vocabSize
|
|
1466
|
-
)
|
|
1467
|
-
: await runMatmul(
|
|
1468
|
-
softmaxTensor,
|
|
1469
|
-
borrowed.softEmbedding.value,
|
|
1470
|
-
canvasLength,
|
|
1471
|
-
hiddenSize,
|
|
1472
|
-
vocabSize,
|
|
1473
|
-
{
|
|
1474
|
-
transposeB: this.#state.embeddingTranspose === true,
|
|
1475
|
-
role: 'diffusion_gemma_self_conditioning_embed',
|
|
1476
|
-
outputDtype: 'f32',
|
|
1477
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1478
|
-
}
|
|
1479
|
-
);
|
|
1480
|
-
}
|
|
1481
|
-
scaledSoftEmbeddings = await runScale(softEmbeddings, Math.sqrt(hiddenSize), {
|
|
1482
|
-
count: elementCount,
|
|
1483
|
-
});
|
|
1484
|
-
if (softEmbeddings !== softEmbeddingStateTensor || softEmbeddingStateOwned) {
|
|
1485
|
-
releaseTensorOnce(softEmbeddings);
|
|
1486
|
-
}
|
|
1487
|
-
softEmbeddings = scaledSoftEmbeddings;
|
|
1488
|
-
scaledSoftEmbeddings = null;
|
|
1489
|
-
} else {
|
|
1490
|
-
const zeroBuffer = acquireBuffer(elementCount * Float32Array.BYTES_PER_ELEMENT, undefined, 'diffusion_gemma_self_conditioning_zero');
|
|
1491
|
-
uploadData(zeroBuffer, new Uint8Array(elementCount * Float32Array.BYTES_PER_ELEMENT));
|
|
1492
|
-
softEmbeddings = createTensor(zeroBuffer, 'f32', [canvasLength, hiddenSize], 'diffusion_gemma_self_conditioning_zero');
|
|
1493
|
-
}
|
|
1494
|
-
|
|
1495
|
-
normedSoft = await runRMSNorm(softEmbeddings, borrowed.preNorm.value, config.rmsNormEps, {
|
|
1496
|
-
batchSize: canvasLength,
|
|
1497
|
-
hiddenSize,
|
|
1498
|
-
rmsNormWeightOffset: false,
|
|
1499
|
-
});
|
|
1500
|
-
const intermediateSize = config.intermediateSize;
|
|
1501
|
-
gate = await runMatmul(
|
|
1502
|
-
normedSoft,
|
|
1503
|
-
borrowed.gateProj.value,
|
|
1504
|
-
canvasLength,
|
|
1505
|
-
intermediateSize,
|
|
1506
|
-
hiddenSize,
|
|
1507
|
-
{
|
|
1508
|
-
transposeB: 'auto',
|
|
1509
|
-
role: 'diffusion_gemma_self_conditioning_gate',
|
|
1510
|
-
outputDtype: 'f32',
|
|
1511
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1512
|
-
}
|
|
1513
|
-
);
|
|
1514
|
-
up = await runMatmul(
|
|
1515
|
-
normedSoft,
|
|
1516
|
-
borrowed.upProj.value,
|
|
1517
|
-
canvasLength,
|
|
1518
|
-
intermediateSize,
|
|
1519
|
-
hiddenSize,
|
|
1520
|
-
{
|
|
1521
|
-
transposeB: 'auto',
|
|
1522
|
-
role: 'diffusion_gemma_self_conditioning_up',
|
|
1523
|
-
outputDtype: 'f32',
|
|
1524
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1525
|
-
}
|
|
1526
|
-
);
|
|
1527
|
-
activated = await runGeLU(up, {
|
|
1528
|
-
size: canvasLength * intermediateSize,
|
|
1529
|
-
gate,
|
|
1530
|
-
});
|
|
1531
|
-
down = await runMatmul(
|
|
1532
|
-
activated,
|
|
1533
|
-
borrowed.downProj.value,
|
|
1534
|
-
canvasLength,
|
|
1535
|
-
hiddenSize,
|
|
1536
|
-
intermediateSize,
|
|
1537
|
-
{
|
|
1538
|
-
transposeB: 'auto',
|
|
1539
|
-
role: 'diffusion_gemma_self_conditioning_down',
|
|
1540
|
-
outputDtype: 'f32',
|
|
1541
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1542
|
-
}
|
|
1543
|
-
);
|
|
1544
|
-
combined = await runResidualAdd(baseEmbeddings, down, elementCount, {
|
|
1545
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1546
|
-
});
|
|
1547
|
-
const postNormWeight = weights.postNorm
|
|
1548
|
-
? borrowNormWeight(weights.postNorm, 'diffusion_gemma_self_conditioning.post_norm')
|
|
1549
|
-
: {
|
|
1550
|
-
value: getQKNormOnesBuffer(hiddenSize),
|
|
1551
|
-
owned: false,
|
|
1552
|
-
};
|
|
1553
|
-
borrowed.postNorm = postNormWeight;
|
|
1554
|
-
output = await runRMSNorm(combined, postNormWeight.value, config.rmsNormEps, {
|
|
1555
|
-
batchSize: canvasLength,
|
|
1556
|
-
hiddenSize,
|
|
1557
|
-
rmsNormWeightOffset: false,
|
|
1558
|
-
});
|
|
1559
|
-
return output;
|
|
1560
|
-
} catch (error) {
|
|
1561
|
-
releaseTensorOnce(output);
|
|
1562
|
-
throw error;
|
|
1563
|
-
} finally {
|
|
1564
|
-
releaseTensorOnce(combined);
|
|
1565
|
-
releaseTensorOnce(down);
|
|
1566
|
-
releaseTensorOnce(activated);
|
|
1567
|
-
releaseTensorOnce(up);
|
|
1568
|
-
releaseTensorOnce(gate);
|
|
1569
|
-
releaseTensorOnce(normedSoft);
|
|
1570
|
-
if (softEmbeddings !== softEmbeddingStateTensor || softEmbeddingStateOwned) {
|
|
1571
|
-
releaseTensorOnce(softEmbeddings);
|
|
1572
|
-
}
|
|
1573
|
-
releaseTensorOnce(softmaxTensor);
|
|
1574
|
-
if (softLogitsOwned) {
|
|
1575
|
-
releaseTensorOnce(softLogitsTensor);
|
|
1576
|
-
}
|
|
1577
|
-
releaseTensorOnce(baseEmbeddings);
|
|
1578
|
-
releaseBorrowedWeight(borrowed.postNorm);
|
|
1579
|
-
releaseBorrowedWeight(borrowed.downProj);
|
|
1580
|
-
releaseBorrowedWeight(borrowed.upProj);
|
|
1581
|
-
releaseBorrowedWeight(borrowed.gateProj);
|
|
1582
|
-
releaseBorrowedWeight(borrowed.preNorm);
|
|
1583
|
-
releaseBorrowedWeight(borrowed.softEmbedding);
|
|
1584
|
-
}
|
|
519
|
+
return _createDiffusionGemmaSelfConditioningEmbeddingsImpl.apply(this, arguments);
|
|
1585
520
|
}
|
|
1586
521
|
|
|
1587
522
|
async _createDiffusionGemmaSelfConditioningSoftEmbeddingState(logitsState, canvasLength, hiddenSize, vocabSize) {
|
|
1588
|
-
|
|
1589
|
-
if (!canUseChunkedSoftEmbeddingLogits(
|
|
1590
|
-
logitsState,
|
|
1591
|
-
embedBufferRaw,
|
|
1592
|
-
this.#state.embeddingTranspose
|
|
1593
|
-
)) {
|
|
1594
|
-
return null;
|
|
1595
|
-
}
|
|
1596
|
-
|
|
1597
|
-
const borrowed = borrowLinearWeight(embedBufferRaw, 'diffusion_gemma_self_conditioning.embed_tokens');
|
|
1598
|
-
let softEmbedding = null;
|
|
1599
|
-
let scaledSoftEmbedding = null;
|
|
1600
|
-
try {
|
|
1601
|
-
const logitsTensor = createTensor(
|
|
1602
|
-
logitsState.logitsBuffer,
|
|
1603
|
-
logitsState.logitsDtype,
|
|
1604
|
-
[canvasLength, vocabSize],
|
|
1605
|
-
'diffusion_gemma_self_conditioning_logits'
|
|
1606
|
-
);
|
|
1607
|
-
softEmbedding = await runSoftEmbeddingLogitsF16(
|
|
1608
|
-
logitsTensor,
|
|
1609
|
-
borrowed.value,
|
|
1610
|
-
canvasLength,
|
|
1611
|
-
hiddenSize,
|
|
1612
|
-
vocabSize,
|
|
1613
|
-
{
|
|
1614
|
-
temperature: logitsState.temperature,
|
|
1615
|
-
chunkRows: resolveDiffusionGemmaSoftEmbeddingChunkRows(this.#state.runtimeConfig),
|
|
1616
|
-
}
|
|
1617
|
-
);
|
|
1618
|
-
scaledSoftEmbedding = await runScale(softEmbedding, Math.sqrt(hiddenSize), {
|
|
1619
|
-
count: canvasLength * hiddenSize,
|
|
1620
|
-
});
|
|
1621
|
-
const returnedBuffer = scaledSoftEmbedding.buffer;
|
|
1622
|
-
return {
|
|
1623
|
-
kind: 'soft_embedding',
|
|
1624
|
-
buffer: returnedBuffer,
|
|
1625
|
-
dtype: scaledSoftEmbedding.dtype,
|
|
1626
|
-
canvasLength,
|
|
1627
|
-
hiddenSize,
|
|
1628
|
-
scaled: true,
|
|
1629
|
-
releaseOnUse: true,
|
|
1630
|
-
release() {
|
|
1631
|
-
releaseBuffer(returnedBuffer);
|
|
1632
|
-
},
|
|
1633
|
-
};
|
|
1634
|
-
} finally {
|
|
1635
|
-
if (softEmbedding?.buffer && softEmbedding.buffer !== scaledSoftEmbedding?.buffer) {
|
|
1636
|
-
releaseBuffer(softEmbedding.buffer);
|
|
1637
|
-
}
|
|
1638
|
-
releaseBorrowedWeight(borrowed);
|
|
1639
|
-
}
|
|
523
|
+
return _createDiffusionGemmaSelfConditioningSoftEmbeddingStateImpl.apply(this, arguments);
|
|
1640
524
|
}
|
|
1641
525
|
|
|
1642
526
|
async computeDiffusionGemmaCanvasLogits(args, options = {}) {
|
|
1643
|
-
|
|
1644
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
1645
|
-
throw new Error('Generation already in progress');
|
|
1646
|
-
}
|
|
1647
|
-
const canvasIds = normalizeCanvasTokenIds(args?.canvas, 'computeDiffusionGemmaCanvasLogits');
|
|
1648
|
-
if (canvasIds.length === 0) {
|
|
1649
|
-
throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasLogits requires at least one canvas token.');
|
|
1650
|
-
}
|
|
1651
|
-
this._assertTokenIdsInRange(canvasIds, 'computeDiffusionGemmaCanvasLogits.canvas');
|
|
1652
|
-
const seqLenBefore = this.#state.currentSeqLen;
|
|
1653
|
-
const opts = {
|
|
1654
|
-
...resolvePrefillOptions(this.#state, {
|
|
1655
|
-
...options,
|
|
1656
|
-
useChatTemplate: false,
|
|
1657
|
-
}),
|
|
1658
|
-
_diffusionGemmaDecoder: true,
|
|
1659
|
-
};
|
|
1660
|
-
let selfConditioned = null;
|
|
1661
|
-
let currentHiddenBuffer = null;
|
|
1662
|
-
try {
|
|
1663
|
-
selfConditioned = await this._createDiffusionGemmaSelfConditioningEmbeddings(
|
|
1664
|
-
canvasIds,
|
|
1665
|
-
args?.selfConditioningLogits ?? null,
|
|
1666
|
-
opts
|
|
1667
|
-
);
|
|
1668
|
-
const prefillResult = await this._prefillToHidden(canvasIds, {
|
|
1669
|
-
...opts,
|
|
1670
|
-
embeddingOverrides: {
|
|
1671
|
-
offset: 0,
|
|
1672
|
-
prefixLength: canvasIds.length,
|
|
1673
|
-
embeddings: selfConditioned.buffer,
|
|
1674
|
-
},
|
|
1675
|
-
});
|
|
1676
|
-
const {
|
|
1677
|
-
numTokens,
|
|
1678
|
-
currentRecorder,
|
|
1679
|
-
recordProfile,
|
|
1680
|
-
debugCheckBuffer,
|
|
1681
|
-
} = prefillResult;
|
|
1682
|
-
currentHiddenBuffer = prefillResult.currentHiddenBuffer;
|
|
1683
|
-
if (currentRecorder) {
|
|
1684
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
1685
|
-
await currentRecorder.submitAndWait();
|
|
1686
|
-
await recordProfile(currentRecorder);
|
|
1687
|
-
} else {
|
|
1688
|
-
const device = getDevice();
|
|
1689
|
-
if (device) {
|
|
1690
|
-
await device.queue.onSubmittedWorkDone();
|
|
1691
|
-
}
|
|
1692
|
-
}
|
|
1693
|
-
const logits = await computeLogits(
|
|
1694
|
-
currentHiddenBuffer,
|
|
1695
|
-
numTokens,
|
|
1696
|
-
getLogitsWeights(this.#state),
|
|
1697
|
-
getLogitsConfig(this.#state),
|
|
1698
|
-
this.#state.useGPU,
|
|
1699
|
-
this.#state.debugFlags,
|
|
1700
|
-
undefined,
|
|
1701
|
-
debugCheckBuffer,
|
|
1702
|
-
this.#state.runtimeConfig.shared.debug.probes,
|
|
1703
|
-
{ lastPositionOnly: false },
|
|
1704
|
-
this.#state.operatorDiagnostics
|
|
1705
|
-
);
|
|
1706
|
-
const expected = canvasIds.length * this.#state.modelConfig.vocabSize;
|
|
1707
|
-
if (logits.length !== expected) {
|
|
1708
|
-
throw new Error(
|
|
1709
|
-
`[DiffusionGemma] canvas logits length mismatch: expected ${expected}, got ${logits.length}.`
|
|
1710
|
-
);
|
|
1711
|
-
}
|
|
1712
|
-
return logits;
|
|
1713
|
-
} finally {
|
|
1714
|
-
this.#state.currentSeqLen = seqLenBefore;
|
|
1715
|
-
if (currentHiddenBuffer) {
|
|
1716
|
-
releaseBuffer(currentHiddenBuffer);
|
|
1717
|
-
}
|
|
1718
|
-
if (selfConditioned?.buffer) {
|
|
1719
|
-
releaseBuffer(selfConditioned.buffer);
|
|
1720
|
-
}
|
|
1721
|
-
}
|
|
527
|
+
return computeDiffusionGemmaCanvasLogitsImpl.apply(this, arguments);
|
|
1722
528
|
}
|
|
1723
529
|
|
|
1724
530
|
async computeDiffusionGemmaCanvasStep(args, options = {}) {
|
|
1725
|
-
|
|
1726
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
1727
|
-
throw new Error('Generation already in progress');
|
|
1728
|
-
}
|
|
1729
|
-
const canvasIds = normalizeCanvasTokenIds(args?.canvas, 'computeDiffusionGemmaCanvasStep');
|
|
1730
|
-
if (canvasIds.length === 0) {
|
|
1731
|
-
throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasStep requires at least one canvas token.');
|
|
1732
|
-
}
|
|
1733
|
-
this._assertTokenIdsInRange(canvasIds, 'computeDiffusionGemmaCanvasStep.canvas');
|
|
1734
|
-
const temperature = args?.temperature;
|
|
1735
|
-
if (typeof temperature !== 'number' || !Number.isFinite(temperature) || temperature <= 0) {
|
|
1736
|
-
throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasStep requires a positive temperature.');
|
|
1737
|
-
}
|
|
1738
|
-
const seqLenBefore = this.#state.currentSeqLen;
|
|
1739
|
-
const opts = {
|
|
1740
|
-
...resolvePrefillOptions(this.#state, {
|
|
1741
|
-
...options,
|
|
1742
|
-
useChatTemplate: false,
|
|
1743
|
-
}),
|
|
1744
|
-
_diffusionGemmaDecoder: true,
|
|
1745
|
-
};
|
|
1746
|
-
let selfConditioned = null;
|
|
1747
|
-
let currentHiddenBuffer = null;
|
|
1748
|
-
let logitsBuffer = null;
|
|
1749
|
-
let statsBuffers = null;
|
|
1750
|
-
try {
|
|
1751
|
-
selfConditioned = await this._createDiffusionGemmaSelfConditioningEmbeddings(
|
|
1752
|
-
canvasIds,
|
|
1753
|
-
args?.selfConditioningLogits ?? null,
|
|
1754
|
-
opts
|
|
1755
|
-
);
|
|
1756
|
-
const prefillResult = await this._prefillToHidden(canvasIds, {
|
|
1757
|
-
...opts,
|
|
1758
|
-
embeddingOverrides: {
|
|
1759
|
-
offset: 0,
|
|
1760
|
-
prefixLength: canvasIds.length,
|
|
1761
|
-
embeddings: selfConditioned.buffer,
|
|
1762
|
-
},
|
|
1763
|
-
});
|
|
1764
|
-
const {
|
|
1765
|
-
numTokens,
|
|
1766
|
-
currentRecorder,
|
|
1767
|
-
recordProfile,
|
|
1768
|
-
debugCheckBuffer,
|
|
1769
|
-
} = prefillResult;
|
|
1770
|
-
currentHiddenBuffer = prefillResult.currentHiddenBuffer;
|
|
1771
|
-
if (currentRecorder) {
|
|
1772
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
1773
|
-
await currentRecorder.submitAndWait();
|
|
1774
|
-
await recordProfile(currentRecorder);
|
|
1775
|
-
} else {
|
|
1776
|
-
const device = getDevice();
|
|
1777
|
-
if (device) {
|
|
1778
|
-
await device.queue.onSubmittedWorkDone();
|
|
1779
|
-
}
|
|
1780
|
-
}
|
|
1781
|
-
if (selfConditioned?.buffer) {
|
|
1782
|
-
releaseBuffer(selfConditioned.buffer);
|
|
1783
|
-
selfConditioned = null;
|
|
1784
|
-
}
|
|
1785
|
-
|
|
1786
|
-
const logitsResult = await computeLogitsGPU(
|
|
1787
|
-
currentHiddenBuffer,
|
|
1788
|
-
numTokens,
|
|
1789
|
-
getLogitsWeights(this.#state),
|
|
1790
|
-
getLogitsConfig(this.#state),
|
|
1791
|
-
this.#state.debugFlags,
|
|
1792
|
-
this.#state.operatorDiagnostics
|
|
1793
|
-
);
|
|
1794
|
-
if (!logitsResult?.logitsBuffer) {
|
|
1795
|
-
throw new Error('[DiffusionGemma] GPU canvas step requires GPU logits.');
|
|
1796
|
-
}
|
|
1797
|
-
logitsBuffer = logitsResult.logitsBuffer;
|
|
1798
|
-
if (currentHiddenBuffer) {
|
|
1799
|
-
releaseBuffer(currentHiddenBuffer);
|
|
1800
|
-
currentHiddenBuffer = null;
|
|
1801
|
-
}
|
|
1802
|
-
if (logitsResult.logitsDtype !== 'f32') {
|
|
1803
|
-
throw new Error(
|
|
1804
|
-
`[DiffusionGemma] GPU canvas stats require f32 logits, got "${logitsResult.logitsDtype}".`
|
|
1805
|
-
);
|
|
1806
|
-
}
|
|
1807
|
-
if (logitsResult.vocabSize !== this.#state.modelConfig.vocabSize) {
|
|
1808
|
-
throw new Error(
|
|
1809
|
-
`[DiffusionGemma] canvas logits vocab mismatch: expected ${this.#state.modelConfig.vocabSize}, ` +
|
|
1810
|
-
`got ${logitsResult.vocabSize}.`
|
|
1811
|
-
);
|
|
1812
|
-
}
|
|
1813
|
-
|
|
1814
|
-
statsBuffers = await runDiffusionGemmaCanvasStats(logitsBuffer, {
|
|
1815
|
-
canvasLength: canvasIds.length,
|
|
1816
|
-
vocabSize: logitsResult.vocabSize,
|
|
1817
|
-
temperature,
|
|
1818
|
-
padTokenId: this.#state.modelConfig.diffusionGemma?.padTokenId ?? null,
|
|
1819
|
-
logitSoftcap: this.#state.modelConfig.finalLogitSoftcapping ?? 0,
|
|
1820
|
-
});
|
|
1821
|
-
const [argmaxData, entropyData] = await Promise.all([
|
|
1822
|
-
readBuffer(statsBuffers.argmaxBuffer, canvasIds.length * Uint32Array.BYTES_PER_ELEMENT),
|
|
1823
|
-
readBuffer(statsBuffers.entropyBuffer, canvasIds.length * Float32Array.BYTES_PER_ELEMENT),
|
|
1824
|
-
]);
|
|
1825
|
-
const argmaxCanvas = Int32Array.from(new Uint32Array(argmaxData));
|
|
1826
|
-
const entropies = new Float32Array(entropyData);
|
|
1827
|
-
const logitsState = {
|
|
1828
|
-
logitsBuffer,
|
|
1829
|
-
logitsDtype: logitsResult.logitsDtype,
|
|
1830
|
-
vocabSize: logitsResult.vocabSize,
|
|
1831
|
-
canvasLength: canvasIds.length,
|
|
1832
|
-
temperature,
|
|
1833
|
-
releaseOnUse: true,
|
|
1834
|
-
};
|
|
1835
|
-
const selfConditioningState =
|
|
1836
|
-
await this._createDiffusionGemmaSelfConditioningSoftEmbeddingState(
|
|
1837
|
-
logitsState,
|
|
1838
|
-
canvasIds.length,
|
|
1839
|
-
this.#state.modelConfig.hiddenSize,
|
|
1840
|
-
logitsResult.vocabSize
|
|
1841
|
-
);
|
|
1842
|
-
if (selfConditioningState) {
|
|
1843
|
-
releaseBuffer(logitsBuffer);
|
|
1844
|
-
logitsBuffer = null;
|
|
1845
|
-
} else {
|
|
1846
|
-
const returnedLogitsBuffer = logitsBuffer;
|
|
1847
|
-
logitsBuffer = null;
|
|
1848
|
-
logitsState.release = () => {
|
|
1849
|
-
releaseBuffer(returnedLogitsBuffer);
|
|
1850
|
-
};
|
|
1851
|
-
}
|
|
1852
|
-
return {
|
|
1853
|
-
argmaxCanvas,
|
|
1854
|
-
entropies,
|
|
1855
|
-
selfConditioningLogits: selfConditioningState ?? logitsState,
|
|
1856
|
-
};
|
|
1857
|
-
} finally {
|
|
1858
|
-
this.#state.currentSeqLen = seqLenBefore;
|
|
1859
|
-
if (statsBuffers?.argmaxBuffer) {
|
|
1860
|
-
releaseBuffer(statsBuffers.argmaxBuffer);
|
|
1861
|
-
}
|
|
1862
|
-
if (statsBuffers?.entropyBuffer) {
|
|
1863
|
-
releaseBuffer(statsBuffers.entropyBuffer);
|
|
1864
|
-
}
|
|
1865
|
-
if (logitsBuffer) {
|
|
1866
|
-
releaseBuffer(logitsBuffer);
|
|
1867
|
-
}
|
|
1868
|
-
if (currentHiddenBuffer) {
|
|
1869
|
-
releaseBuffer(currentHiddenBuffer);
|
|
1870
|
-
}
|
|
1871
|
-
if (selfConditioned?.buffer) {
|
|
1872
|
-
releaseBuffer(selfConditioned.buffer);
|
|
1873
|
-
}
|
|
1874
|
-
}
|
|
531
|
+
return computeDiffusionGemmaCanvasStepImpl.apply(this, arguments);
|
|
1875
532
|
}
|
|
1876
533
|
|
|
1877
534
|
async *generate(prompt, options = {}) {
|
|
@@ -1883,873 +540,35 @@ export class PipelineGenerator {
|
|
|
1883
540
|
}
|
|
1884
541
|
|
|
1885
542
|
async generateTokenIds(prompt, options = {}) {
|
|
1886
|
-
|
|
1887
|
-
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
1888
|
-
|
|
1889
|
-
validateCallTimeOptions(options);
|
|
1890
|
-
|
|
1891
|
-
this.#state.isGenerating = true;
|
|
1892
|
-
this._resetDecodeRuntimeState();
|
|
1893
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
1894
|
-
this.#state.stats.gpuTimeDecodeMs = undefined;
|
|
1895
|
-
this.#state.stats.decodeRecordMs = 0;
|
|
1896
|
-
this.#state.stats.decodeRecordOps = 0;
|
|
1897
|
-
this.#state.stats.decodeRecordPasses = 0;
|
|
1898
|
-
this.#state.stats.decodeRecordOpLabels = {};
|
|
1899
|
-
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
1900
|
-
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
1901
|
-
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
1902
|
-
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
1903
|
-
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
1904
|
-
this.#state.stats.prefillRecordMs = 0;
|
|
1905
|
-
this.#state.stats.prefillRecordOps = 0;
|
|
1906
|
-
this.#state.stats.prefillRecordPasses = 0;
|
|
1907
|
-
this.#state.stats.prefillRecordOpLabels = {};
|
|
1908
|
-
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
1909
|
-
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
1910
|
-
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
1911
|
-
this.#state.stats.singleTokenReadbackMapWaitMs = 0;
|
|
1912
|
-
this.#state.stats.singleTokenReadbackCleanupMs = 0;
|
|
1913
|
-
this.#state.stats.singleTokenReadbackCopyMs = 0;
|
|
1914
|
-
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
1915
|
-
this.#state.stats.pleHotVocabularyHits = 0;
|
|
1916
|
-
this.#state.stats.pleHotVocabularyMisses = 0;
|
|
1917
|
-
this.#state.stats.plePreparedTokenCacheHits = 0;
|
|
1918
|
-
this.#state.stats.plePreparedTokenCacheMisses = 0;
|
|
1919
|
-
this.#state.stats.plePreparedTokenCacheEntries = 0;
|
|
1920
|
-
this.#state.stats.plePreparedTokenCacheBytes = 0;
|
|
1921
|
-
this.#state.stats.decodeMode = null;
|
|
1922
|
-
this.#state.stats.batchGuardReason = null;
|
|
1923
|
-
this.#state.stats.stopReason = null;
|
|
1924
|
-
this.#state.stats.stopTokenId = null;
|
|
1925
|
-
this.#state.stats.ttftMs = 0;
|
|
1926
|
-
const startTime = performance.now();
|
|
1927
|
-
const opts = resolveGenerateOptions(this.#state, options);
|
|
1928
|
-
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
1929
|
-
// Validate and normalize sampling parameters through single source of truth
|
|
1930
|
-
const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
|
|
1931
|
-
opts.temperature = samplingConfig.temperature;
|
|
1932
|
-
opts.topP = samplingConfig.topP;
|
|
1933
|
-
opts.topK = samplingConfig.topK;
|
|
1934
|
-
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
1935
|
-
opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this.#state, samplingConfig);
|
|
1936
|
-
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
1937
|
-
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
1938
|
-
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
1939
|
-
if (diagnosticsEnabled || tsirFixtureCfg) {
|
|
1940
|
-
const captureConfig = {
|
|
1941
|
-
...createDefaultCaptureConfig(),
|
|
1942
|
-
enabled: true,
|
|
1943
|
-
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
1944
|
-
...(options?.diagnostics?.captureConfig ?? {}),
|
|
1945
|
-
};
|
|
1946
|
-
validateCaptureConfig(captureConfig);
|
|
1947
|
-
this.#state.operatorDiagnostics = {
|
|
1948
|
-
enabled: diagnosticsEnabled === true,
|
|
1949
|
-
captureConfig,
|
|
1950
|
-
emitter: diagnosticsEnabled ? new OperatorEventEmitter({
|
|
1951
|
-
modelHash: this.#state.manifest?.modelId ?? null,
|
|
1952
|
-
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
1953
|
-
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
1954
|
-
}) : null,
|
|
1955
|
-
tsirFixture: tsirFixtureCfg ? {
|
|
1956
|
-
dir: tsirFixtureCfg.dir,
|
|
1957
|
-
layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
|
|
1958
|
-
? tsirFixtureCfg.layerFilter
|
|
1959
|
-
: null,
|
|
1960
|
-
records: [],
|
|
1961
|
-
} : null,
|
|
1962
|
-
};
|
|
1963
|
-
}
|
|
1964
|
-
|
|
1965
|
-
try {
|
|
1966
|
-
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
1967
|
-
const prefillStart = performance.now();
|
|
1968
|
-
const { inputIds, logits: initialPrefillLogits } = await this._prefillPromptToLogits(prompt, opts, 'generateTokenIds');
|
|
1969
|
-
let prefillLogits = initialPrefillLogits;
|
|
1970
|
-
this.#state.stats.prefillTimeMs = performance.now() - prefillStart;
|
|
1971
|
-
this._assertTokenIdsInRange(inputIds, 'generateTokenIds.prefillTokens');
|
|
1972
|
-
const generatedIds = [...inputIds];
|
|
1973
|
-
this.#state.stats.prefillTokens = inputIds.length;
|
|
1974
|
-
|
|
1975
|
-
let firstToken;
|
|
1976
|
-
try {
|
|
1977
|
-
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
1978
|
-
} catch (error) {
|
|
1979
|
-
if (!this._shouldUseFinitenessFallback(error, 'prefill-sample')) {
|
|
1980
|
-
throw error;
|
|
1981
|
-
}
|
|
1982
|
-
prefillLogits = await this._retryWithPersistentFinitenessFallback(
|
|
1983
|
-
opts,
|
|
1984
|
-
'prefill-sample',
|
|
1985
|
-
opts.maxTokens,
|
|
1986
|
-
() => this._prefill(inputIds, opts),
|
|
1987
|
-
prefillStartSeqLen
|
|
1988
|
-
);
|
|
1989
|
-
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
1990
|
-
}
|
|
1991
|
-
|
|
1992
|
-
const stopTokenIds = this.#state.modelConfig.stopTokenIds;
|
|
1993
|
-
const eosToken = this.#state.tokenizer.getSpecialTokens?.()?.eos;
|
|
1994
|
-
const stopSequenceStart = inputIds.length;
|
|
1995
|
-
generatedIds.push(firstToken);
|
|
1996
|
-
const tokenIds = [firstToken];
|
|
1997
|
-
this.#state.stats.ttftMs = performance.now() - startTime;
|
|
1998
|
-
markKernelCacheWarmed();
|
|
1999
|
-
|
|
2000
|
-
const decodeRuntime = {
|
|
2001
|
-
stopTokenIds,
|
|
2002
|
-
eosToken,
|
|
2003
|
-
stopSequenceStart,
|
|
2004
|
-
decodeToken: () => '',
|
|
2005
|
-
emitMode: 'token',
|
|
2006
|
-
};
|
|
2007
|
-
|
|
2008
|
-
if (!this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
|
|
2009
|
-
for await (const tokenId of this._runDecodeLoop(generatedIds, opts, options, decodeRuntime)) {
|
|
2010
|
-
tokenIds.push(tokenId);
|
|
2011
|
-
}
|
|
2012
|
-
} else {
|
|
2013
|
-
this._recordStopReason('stop-token-or-sequence', firstToken);
|
|
2014
|
-
this.#state.stats.decodeTimeMs = 0;
|
|
2015
|
-
this.#state.stats.tokensGenerated = 1;
|
|
2016
|
-
this.#state.stats.decodeTokens = 1;
|
|
2017
|
-
}
|
|
2018
|
-
|
|
2019
|
-
this.#state.stats.totalTimeMs = performance.now() - startTime;
|
|
2020
|
-
|
|
2021
|
-
return {
|
|
2022
|
-
tokenIds,
|
|
2023
|
-
stats: this.#state.stats,
|
|
2024
|
-
};
|
|
2025
|
-
} finally {
|
|
2026
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
2027
|
-
if (this.#state.stats.executionPlan) {
|
|
2028
|
-
this.#state.stats.executionPlan.finalActivePlanId = this.#state.executionPlanState?.activePlanId ?? null;
|
|
2029
|
-
}
|
|
2030
|
-
resetActiveExecutionPlan(this.#state);
|
|
2031
|
-
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
2032
|
-
? {
|
|
2033
|
-
enabled: true,
|
|
2034
|
-
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
2035
|
-
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
2036
|
-
}
|
|
2037
|
-
: null;
|
|
2038
|
-
this.#state.operatorDiagnostics = null;
|
|
2039
|
-
this.#state.isGenerating = false;
|
|
2040
|
-
}
|
|
543
|
+
return generateTokenIdsImpl.apply(this, arguments);
|
|
2041
544
|
}
|
|
2042
545
|
|
|
2043
|
-
|
|
2044
546
|
async prefillKVOnly(prompt, options = {}) {
|
|
2045
|
-
|
|
2046
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2047
|
-
throw new Error('Generation already in progress');
|
|
2048
|
-
}
|
|
2049
|
-
assertIncrementalDecodeSupport(this.#state, 'prefillKVOnly');
|
|
2050
|
-
this._resetDecodeRuntimeState();
|
|
2051
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2052
|
-
this.#state.stats.prefillProfileSteps = [];
|
|
2053
|
-
const opts = resolvePrefillOptions(this.#state, options);
|
|
2054
|
-
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
2055
|
-
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillKVOnly', opts.inputIds);
|
|
2056
|
-
if (opts.debug) {
|
|
2057
|
-
log.debug('Pipeline', `PrefillKVOnly: ${inputIds.length} tokens`);
|
|
2058
|
-
}
|
|
2059
|
-
|
|
2060
|
-
try {
|
|
2061
|
-
let prefillResult;
|
|
2062
|
-
try {
|
|
2063
|
-
prefillResult = await this._prefillToHidden(inputIds, opts);
|
|
2064
|
-
} catch (error) {
|
|
2065
|
-
if (this._shouldUseFinitenessFallback(error, 'prefillKVOnly')) {
|
|
2066
|
-
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillKVOnly. Retrying with F32 precision.`);
|
|
2067
|
-
prefillResult = await this._retryWithPersistentFinitenessFallback(
|
|
2068
|
-
opts,
|
|
2069
|
-
'prefillKVOnly',
|
|
2070
|
-
1,
|
|
2071
|
-
() => this._prefillToHidden(inputIds, opts),
|
|
2072
|
-
prefillStartSeqLen
|
|
2073
|
-
);
|
|
2074
|
-
} else {
|
|
2075
|
-
throw error;
|
|
2076
|
-
}
|
|
2077
|
-
}
|
|
2078
|
-
|
|
2079
|
-
const {
|
|
2080
|
-
numTokens,
|
|
2081
|
-
startPos,
|
|
2082
|
-
currentRecorder,
|
|
2083
|
-
recordProfile,
|
|
2084
|
-
currentHiddenBuffer,
|
|
2085
|
-
} = prefillResult;
|
|
2086
|
-
|
|
2087
|
-
// Ensure prefill work completes before returning a usable snapshot.
|
|
2088
|
-
if (currentRecorder) {
|
|
2089
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
2090
|
-
await currentRecorder.submitAndWait();
|
|
2091
|
-
await recordProfile(currentRecorder);
|
|
2092
|
-
} else {
|
|
2093
|
-
const device = getDevice();
|
|
2094
|
-
if (device) {
|
|
2095
|
-
await device.queue.onSubmittedWorkDone();
|
|
2096
|
-
}
|
|
2097
|
-
}
|
|
2098
|
-
|
|
2099
|
-
this.#state.currentSeqLen = startPos + numTokens;
|
|
2100
|
-
releaseBuffer(currentHiddenBuffer);
|
|
2101
|
-
|
|
2102
|
-
const snapshot = this.#state.kvCache?.clone();
|
|
2103
|
-
if (!snapshot) {
|
|
2104
|
-
throw new Error('KV cache unavailable after prefill');
|
|
2105
|
-
}
|
|
2106
|
-
|
|
2107
|
-
return {
|
|
2108
|
-
cache: snapshot,
|
|
2109
|
-
seqLen: this.#state.currentSeqLen,
|
|
2110
|
-
tokens: inputIds,
|
|
2111
|
-
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
2112
|
-
};
|
|
2113
|
-
} finally {
|
|
2114
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
2115
|
-
}
|
|
547
|
+
return prefillKVOnlyImpl.apply(this, arguments);
|
|
2116
548
|
}
|
|
2117
549
|
|
|
2118
550
|
async prefillForLoRATraining(inputIds, options = {}) {
|
|
2119
|
-
|
|
2120
|
-
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
2121
|
-
if (!Array.isArray(inputIds) && !(inputIds instanceof Int32Array) && !(inputIds instanceof Uint32Array)) {
|
|
2122
|
-
throw new Error('prefillForLoRATraining requires token IDs.');
|
|
2123
|
-
}
|
|
2124
|
-
const tokens = Array.from(inputIds, (value) => Number(value));
|
|
2125
|
-
if (tokens.length === 0) throw new Error('prefillForLoRATraining requires at least one token.');
|
|
2126
|
-
this._assertTokenIdsInRange(tokens, 'prefillForLoRATraining.inputIds');
|
|
2127
|
-
|
|
2128
|
-
const config = this.#state.modelConfig;
|
|
2129
|
-
const layerIdx = Number(options.layerIdx);
|
|
2130
|
-
if (!Number.isInteger(layerIdx) || layerIdx !== config.numLayers - 1) {
|
|
2131
|
-
throw new Error(`native_lora_target_not_supported: layerIdx must be the final layer (${config.numLayers - 1}).`);
|
|
2132
|
-
}
|
|
2133
|
-
if (options.module !== 'down_proj') {
|
|
2134
|
-
throw new Error('native_lora_target_not_supported: module must be down_proj.');
|
|
2135
|
-
}
|
|
2136
|
-
if (config.layerTypes?.[layerIdx] !== 'full_attention') {
|
|
2137
|
-
throw new Error('native_lora_target_not_supported: final layer must be full_attention.');
|
|
2138
|
-
}
|
|
2139
|
-
if (config.postFeedforwardNorm === true) {
|
|
2140
|
-
throw new Error('native_lora_target_not_supported: final-layer post-feedforward normalization is not supported.');
|
|
2141
|
-
}
|
|
2142
|
-
if (this.#state.lora) {
|
|
2143
|
-
throw new Error('native_lora_base_capture_requires_inactive_adapter.');
|
|
2144
|
-
}
|
|
2145
|
-
|
|
2146
|
-
this.resetToSeqLen(0);
|
|
2147
|
-
this._resetDecodeRuntimeState();
|
|
2148
|
-
const opts = resolvePrefillOptions(this.#state, {
|
|
2149
|
-
...options,
|
|
2150
|
-
inputIds: tokens,
|
|
2151
|
-
useChatTemplate: false,
|
|
2152
|
-
});
|
|
2153
|
-
let activationTensor = null;
|
|
2154
|
-
let hiddenTensor = null;
|
|
2155
|
-
let disposed = false;
|
|
2156
|
-
const releaseOwned = () => {
|
|
2157
|
-
if (disposed) return;
|
|
2158
|
-
disposed = true;
|
|
2159
|
-
if (activationTensor?.buffer) releaseBuffer(activationTensor.buffer);
|
|
2160
|
-
if (hiddenTensor?.buffer) releaseBuffer(hiddenTensor.buffer);
|
|
2161
|
-
};
|
|
2162
|
-
const capture = async ({ tensor, numTokens, hiddenSize, recorder }) => {
|
|
2163
|
-
if (activationTensor) {
|
|
2164
|
-
throw new Error('native_lora_training_capture_duplicated.');
|
|
2165
|
-
}
|
|
2166
|
-
const bytesPerElement = tensor.dtype === 'f16' ? 2 : 4;
|
|
2167
|
-
const byteLength = Math.ceil((numTokens * hiddenSize * bytesPerElement) / 4) * 4;
|
|
2168
|
-
const buffer = acquireBuffer(byteLength, undefined, `L${layerIdx}.native_lora_ffn_act`);
|
|
2169
|
-
if (recorder) {
|
|
2170
|
-
recorder.getEncoder().copyBufferToBuffer(tensor.buffer, 0, buffer, 0, byteLength);
|
|
2171
|
-
} else {
|
|
2172
|
-
const device = getDevice();
|
|
2173
|
-
const encoder = device.createCommandEncoder({ label: `L${layerIdx}.native_lora_capture` });
|
|
2174
|
-
encoder.copyBufferToBuffer(tensor.buffer, 0, buffer, 0, byteLength);
|
|
2175
|
-
device.queue.submit([encoder.finish()]);
|
|
2176
|
-
}
|
|
2177
|
-
activationTensor = createTensor(
|
|
2178
|
-
buffer,
|
|
2179
|
-
tensor.dtype,
|
|
2180
|
-
[numTokens, hiddenSize],
|
|
2181
|
-
`L${layerIdx}.native_lora_ffn_act`
|
|
2182
|
-
);
|
|
2183
|
-
};
|
|
2184
|
-
|
|
2185
|
-
try {
|
|
2186
|
-
const prefill = await this._prefillToHidden(tokens, {
|
|
2187
|
-
...opts,
|
|
2188
|
-
_trainingCapture: { layerIdx, stage: 'ffn_act', capture },
|
|
2189
|
-
});
|
|
2190
|
-
if (prefill.currentRecorder) {
|
|
2191
|
-
recordPrefillRecorderStats(this.#state, prefill.currentRecorder);
|
|
2192
|
-
await prefill.currentRecorder.submitAndWait();
|
|
2193
|
-
await prefill.recordProfile(prefill.currentRecorder);
|
|
2194
|
-
} else {
|
|
2195
|
-
await getDevice().queue.onSubmittedWorkDone();
|
|
2196
|
-
}
|
|
2197
|
-
if (!activationTensor) {
|
|
2198
|
-
releaseBuffer(prefill.currentHiddenBuffer);
|
|
2199
|
-
throw new Error('native_lora_training_activation_not_captured.');
|
|
2200
|
-
}
|
|
2201
|
-
hiddenTensor = createTensor(
|
|
2202
|
-
prefill.currentHiddenBuffer,
|
|
2203
|
-
prefill.activationDtype,
|
|
2204
|
-
[prefill.numTokens, config.hiddenSize],
|
|
2205
|
-
`L${layerIdx}.native_lora_base_hidden`
|
|
2206
|
-
);
|
|
2207
|
-
this.resetToSeqLen(0);
|
|
2208
|
-
return {
|
|
2209
|
-
inputIds: tokens,
|
|
2210
|
-
layerIdx,
|
|
2211
|
-
module: 'down_proj',
|
|
2212
|
-
activation: activationTensor,
|
|
2213
|
-
baseHidden: hiddenTensor,
|
|
2214
|
-
dispose: releaseOwned,
|
|
2215
|
-
};
|
|
2216
|
-
} catch (error) {
|
|
2217
|
-
releaseOwned();
|
|
2218
|
-
this.resetToSeqLen(0);
|
|
2219
|
-
throw error;
|
|
2220
|
-
}
|
|
551
|
+
return prefillForLoRATrainingImpl.apply(this, arguments);
|
|
2221
552
|
}
|
|
2222
553
|
|
|
2223
554
|
async prefillWithEmbedding(prompt, options = {}) {
|
|
2224
|
-
|
|
2225
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2226
|
-
throw new Error('Generation already in progress');
|
|
2227
|
-
}
|
|
2228
|
-
assertIncrementalDecodeSupport(this.#state, 'prefillWithEmbedding');
|
|
2229
|
-
const totalStart = performance.now();
|
|
2230
|
-
this._resetDecodeRuntimeState();
|
|
2231
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2232
|
-
this.#state.stats.prefillProfileSteps = [];
|
|
2233
|
-
const opts = resolvePrefillEmbeddingOptions(this.#state, options);
|
|
2234
|
-
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
2235
|
-
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
2236
|
-
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
2237
|
-
if (diagnosticsEnabled || tsirFixtureCfg) {
|
|
2238
|
-
const captureConfig = {
|
|
2239
|
-
...createDefaultCaptureConfig(),
|
|
2240
|
-
enabled: true,
|
|
2241
|
-
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
2242
|
-
...(options?.diagnostics?.captureConfig ?? {}),
|
|
2243
|
-
};
|
|
2244
|
-
validateCaptureConfig(captureConfig);
|
|
2245
|
-
this.#state.operatorDiagnostics = {
|
|
2246
|
-
enabled: diagnosticsEnabled === true,
|
|
2247
|
-
captureConfig,
|
|
2248
|
-
emitter: diagnosticsEnabled ? new OperatorEventEmitter({
|
|
2249
|
-
modelHash: this.#state.manifest?.modelId ?? null,
|
|
2250
|
-
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
2251
|
-
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
2252
|
-
}) : null,
|
|
2253
|
-
tsirFixture: tsirFixtureCfg ? {
|
|
2254
|
-
dir: tsirFixtureCfg.dir,
|
|
2255
|
-
layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
|
|
2256
|
-
? tsirFixtureCfg.layerFilter
|
|
2257
|
-
: null,
|
|
2258
|
-
records: [],
|
|
2259
|
-
} : null,
|
|
2260
|
-
};
|
|
2261
|
-
}
|
|
2262
|
-
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
2263
|
-
const inputStart = performance.now();
|
|
2264
|
-
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
|
|
2265
|
-
const inputMs = performance.now() - inputStart;
|
|
2266
|
-
if (opts.debug) {
|
|
2267
|
-
log.debug('Pipeline', `PrefillWithEmbedding: ${inputIds.length} tokens (mode=${opts.embeddingMode})`);
|
|
2268
|
-
}
|
|
2269
|
-
|
|
2270
|
-
try {
|
|
2271
|
-
let prefillResult;
|
|
2272
|
-
const prefillStart = performance.now();
|
|
2273
|
-
try {
|
|
2274
|
-
prefillResult = await this._prefillToHidden(inputIds, {
|
|
2275
|
-
...opts,
|
|
2276
|
-
_embeddingOnly: options.__skipStateSnapshot === true,
|
|
2277
|
-
});
|
|
2278
|
-
} catch (error) {
|
|
2279
|
-
if (shouldRetryWithFinitenessFallback(error)) {
|
|
2280
|
-
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillWithEmbedding. Retrying with F32 precision.`);
|
|
2281
|
-
prefillResult = await this._retryWithPersistentFinitenessFallback(
|
|
2282
|
-
opts,
|
|
2283
|
-
'prefillWithEmbedding',
|
|
2284
|
-
1,
|
|
2285
|
-
() => this._prefillToHidden(inputIds, {
|
|
2286
|
-
...opts,
|
|
2287
|
-
_embeddingOnly: options.__skipStateSnapshot === true,
|
|
2288
|
-
}),
|
|
2289
|
-
prefillStartSeqLen
|
|
2290
|
-
);
|
|
2291
|
-
} else {
|
|
2292
|
-
throw error;
|
|
2293
|
-
}
|
|
2294
|
-
}
|
|
2295
|
-
const prefillMs = performance.now() - prefillStart;
|
|
2296
|
-
|
|
2297
|
-
const {
|
|
2298
|
-
numTokens,
|
|
2299
|
-
config,
|
|
2300
|
-
startPos,
|
|
2301
|
-
activationDtype,
|
|
2302
|
-
activationBytes,
|
|
2303
|
-
currentRecorder,
|
|
2304
|
-
recordProfile,
|
|
2305
|
-
currentHiddenBuffer,
|
|
2306
|
-
} = prefillResult;
|
|
2307
|
-
|
|
2308
|
-
// Ensure prefill work completes before readback.
|
|
2309
|
-
let submitWaitMs = 0;
|
|
2310
|
-
const submitWaitStart = performance.now();
|
|
2311
|
-
if (currentRecorder) {
|
|
2312
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
2313
|
-
await currentRecorder.submitAndWait();
|
|
2314
|
-
await recordProfile(currentRecorder);
|
|
2315
|
-
} else {
|
|
2316
|
-
const device = getDevice();
|
|
2317
|
-
if (device) {
|
|
2318
|
-
await device.queue.onSubmittedWorkDone();
|
|
2319
|
-
}
|
|
2320
|
-
}
|
|
2321
|
-
submitWaitMs = performance.now() - submitWaitStart;
|
|
2322
|
-
|
|
2323
|
-
if (!allowReadback('pipeline.prefill.embedding')) {
|
|
2324
|
-
throw new Error('GPU readback disabled; cannot return embedding');
|
|
2325
|
-
}
|
|
2326
|
-
|
|
2327
|
-
let embedding;
|
|
2328
|
-
let tokenEmbeddings = null;
|
|
2329
|
-
let logits = null;
|
|
2330
|
-
let hiddenBytes = 0;
|
|
2331
|
-
let readbackMs = 0;
|
|
2332
|
-
let decodeHiddenMs = 0;
|
|
2333
|
-
let finalNormMs = 0;
|
|
2334
|
-
let extractMs = 0;
|
|
2335
|
-
let logitsMs = 0;
|
|
2336
|
-
try {
|
|
2337
|
-
if (options.__returnSequenceLogits === true) {
|
|
2338
|
-
const logitsStart = performance.now();
|
|
2339
|
-
logits = await computeLogits(
|
|
2340
|
-
currentHiddenBuffer,
|
|
2341
|
-
numTokens,
|
|
2342
|
-
getLogitsWeights(this.#state),
|
|
2343
|
-
getLogitsConfig(this.#state),
|
|
2344
|
-
this.#state.useGPU,
|
|
2345
|
-
this.#state.debugFlags,
|
|
2346
|
-
undefined,
|
|
2347
|
-
undefined,
|
|
2348
|
-
this.#state.runtimeConfig.shared.debug.probes,
|
|
2349
|
-
{ lastPositionOnly: false },
|
|
2350
|
-
this.#state.operatorDiagnostics
|
|
2351
|
-
);
|
|
2352
|
-
logitsMs = performance.now() - logitsStart;
|
|
2353
|
-
}
|
|
2354
|
-
const hiddenSize = config.hiddenSize;
|
|
2355
|
-
hiddenBytes = numTokens * hiddenSize * activationBytes;
|
|
2356
|
-
const readbackStart = performance.now();
|
|
2357
|
-
const hiddenData = await readBuffer(currentHiddenBuffer, hiddenBytes);
|
|
2358
|
-
readbackMs = performance.now() - readbackStart;
|
|
2359
|
-
if (hiddenData.byteLength === 0) {
|
|
2360
|
-
throw new Error('GPU readback disabled; cannot return embedding');
|
|
2361
|
-
}
|
|
2362
|
-
const decodeStart = performance.now();
|
|
2363
|
-
const hiddenStates = decodeReadback(hiddenData, activationDtype);
|
|
2364
|
-
decodeHiddenMs = performance.now() - decodeStart;
|
|
2365
|
-
const finalNormStart = performance.now();
|
|
2366
|
-
const finalNormWeights = await this._getFinalNormWeights();
|
|
2367
|
-
finalNormMs = performance.now() - finalNormStart;
|
|
2368
|
-
const extractStart = performance.now();
|
|
2369
|
-
if (options.__returnTokenEmbeddings === true) {
|
|
2370
|
-
tokenEmbeddings = this._extractTokenEmbeddingsFromHidden(
|
|
2371
|
-
hiddenStates,
|
|
2372
|
-
numTokens,
|
|
2373
|
-
hiddenSize,
|
|
2374
|
-
finalNormWeights,
|
|
2375
|
-
config
|
|
2376
|
-
);
|
|
2377
|
-
}
|
|
2378
|
-
embedding = extractEmbeddingFromHidden(
|
|
2379
|
-
hiddenStates,
|
|
2380
|
-
numTokens,
|
|
2381
|
-
hiddenSize,
|
|
2382
|
-
opts.embeddingMode,
|
|
2383
|
-
finalNormWeights,
|
|
2384
|
-
config,
|
|
2385
|
-
this.#state.embeddingPostprocessor,
|
|
2386
|
-
tokenEmbeddings,
|
|
2387
|
-
this.#state.weights.get('final_norm_bias') ?? null
|
|
2388
|
-
);
|
|
2389
|
-
extractMs = performance.now() - extractStart;
|
|
2390
|
-
} finally {
|
|
2391
|
-
releaseBuffer(currentHiddenBuffer);
|
|
2392
|
-
}
|
|
2393
|
-
|
|
2394
|
-
this.#state.currentSeqLen = startPos + numTokens;
|
|
2395
|
-
const phase = {
|
|
2396
|
-
totalMs: performance.now() - totalStart,
|
|
2397
|
-
inputMs,
|
|
2398
|
-
prefillMs,
|
|
2399
|
-
submitWaitMs,
|
|
2400
|
-
readbackMs,
|
|
2401
|
-
decodeHiddenMs,
|
|
2402
|
-
finalNormMs,
|
|
2403
|
-
extractMs,
|
|
2404
|
-
logitsMs,
|
|
2405
|
-
hiddenBytes,
|
|
2406
|
-
tokens: numTokens,
|
|
2407
|
-
activationDtype,
|
|
2408
|
-
prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
|
|
2409
|
-
prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
|
|
2410
|
-
prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
|
|
2411
|
-
prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
|
|
2412
|
-
prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
|
|
2413
|
-
gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
|
|
2414
|
-
};
|
|
2415
|
-
|
|
2416
|
-
// Batch embedding skips expensive KV cache clone and linear attention clone
|
|
2417
|
-
// since the caller will reset immediately after extracting the embedding.
|
|
2418
|
-
if (options.__skipStateSnapshot) {
|
|
2419
|
-
return {
|
|
2420
|
-
cache: null,
|
|
2421
|
-
seqLen: this.#state.currentSeqLen,
|
|
2422
|
-
tokens: inputIds,
|
|
2423
|
-
embedding,
|
|
2424
|
-
tokenEmbeddings,
|
|
2425
|
-
logits,
|
|
2426
|
-
embeddingMode: opts.embeddingMode,
|
|
2427
|
-
phase,
|
|
2428
|
-
linearAttention: null,
|
|
2429
|
-
};
|
|
2430
|
-
}
|
|
2431
|
-
|
|
2432
|
-
const snapshot = this.#state.kvCache?.clone();
|
|
2433
|
-
if (!snapshot) {
|
|
2434
|
-
throw new Error('KV cache unavailable after prefill');
|
|
2435
|
-
}
|
|
2436
|
-
|
|
2437
|
-
return {
|
|
2438
|
-
cache: snapshot,
|
|
2439
|
-
seqLen: this.#state.currentSeqLen,
|
|
2440
|
-
tokens: inputIds,
|
|
2441
|
-
embedding,
|
|
2442
|
-
tokenEmbeddings,
|
|
2443
|
-
logits,
|
|
2444
|
-
embeddingMode: opts.embeddingMode,
|
|
2445
|
-
phase,
|
|
2446
|
-
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
2447
|
-
};
|
|
2448
|
-
} finally {
|
|
2449
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
2450
|
-
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
2451
|
-
? {
|
|
2452
|
-
enabled: true,
|
|
2453
|
-
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
2454
|
-
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
2455
|
-
}
|
|
2456
|
-
: null;
|
|
2457
|
-
this.#state.operatorDiagnostics = null;
|
|
2458
|
-
}
|
|
555
|
+
return prefillWithEmbeddingImpl.apply(this, arguments);
|
|
2459
556
|
}
|
|
2460
557
|
|
|
2461
558
|
async prefillWithLogits(prompt, options = {}) {
|
|
2462
|
-
|
|
2463
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2464
|
-
throw new Error('Generation already in progress');
|
|
2465
|
-
}
|
|
2466
|
-
assertIncrementalDecodeSupport(this.#state, 'prefillWithLogits');
|
|
2467
|
-
const totalStart = performance.now();
|
|
2468
|
-
this._resetDecodeRuntimeState();
|
|
2469
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2470
|
-
this.#state.stats.prefillProfileSteps = [];
|
|
2471
|
-
const opts = resolvePrefillOptions(this.#state, options);
|
|
2472
|
-
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
2473
|
-
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
2474
|
-
if (diagnosticsEnabled) {
|
|
2475
|
-
const captureConfig = {
|
|
2476
|
-
...createDefaultCaptureConfig(),
|
|
2477
|
-
enabled: true,
|
|
2478
|
-
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
2479
|
-
...(options?.diagnostics?.captureConfig ?? {}),
|
|
2480
|
-
};
|
|
2481
|
-
validateCaptureConfig(captureConfig);
|
|
2482
|
-
this.#state.operatorDiagnostics = {
|
|
2483
|
-
enabled: true,
|
|
2484
|
-
captureConfig,
|
|
2485
|
-
emitter: new OperatorEventEmitter({
|
|
2486
|
-
modelHash: this.#state.manifest?.modelId ?? null,
|
|
2487
|
-
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
2488
|
-
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
2489
|
-
}),
|
|
2490
|
-
tsirFixture: null,
|
|
2491
|
-
};
|
|
2492
|
-
}
|
|
2493
|
-
try {
|
|
2494
|
-
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
|
|
2495
|
-
|
|
2496
|
-
const snapshot = this.#state.kvCache?.clone();
|
|
2497
|
-
if (!snapshot) {
|
|
2498
|
-
throw new Error('KV cache unavailable after prefill');
|
|
2499
|
-
}
|
|
2500
|
-
|
|
2501
|
-
return {
|
|
2502
|
-
cache: snapshot,
|
|
2503
|
-
seqLen: this.#state.currentSeqLen,
|
|
2504
|
-
tokens: inputIds,
|
|
2505
|
-
logits,
|
|
2506
|
-
phase: {
|
|
2507
|
-
...(prefillPhase ?? {}),
|
|
2508
|
-
totalMs: performance.now() - totalStart,
|
|
2509
|
-
},
|
|
2510
|
-
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
2511
|
-
};
|
|
2512
|
-
} finally {
|
|
2513
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
2514
|
-
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
2515
|
-
? {
|
|
2516
|
-
enabled: true,
|
|
2517
|
-
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
2518
|
-
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
2519
|
-
}
|
|
2520
|
-
: null;
|
|
2521
|
-
this.#state.operatorDiagnostics = null;
|
|
2522
|
-
}
|
|
559
|
+
return prefillWithLogitsImpl.apply(this, arguments);
|
|
2523
560
|
}
|
|
2524
561
|
|
|
2525
562
|
async prefillWithTokenLogits(prompt, tokenIds, options = {}) {
|
|
2526
|
-
|
|
2527
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2528
|
-
throw new Error('Generation already in progress');
|
|
2529
|
-
}
|
|
2530
|
-
assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogits');
|
|
2531
|
-
const totalStart = performance.now();
|
|
2532
|
-
const selectedTokenIds = normalizeSelectedLogitTokenIds(
|
|
2533
|
-
tokenIds,
|
|
2534
|
-
this.#state.modelConfig.vocabSize,
|
|
2535
|
-
'prefillWithTokenLogits.tokenIds'
|
|
2536
|
-
);
|
|
2537
|
-
this._resetDecodeRuntimeState();
|
|
2538
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2539
|
-
this.#state.stats.prefillProfileSteps = [];
|
|
2540
|
-
const opts = resolvePrefillOptions(this.#state, options);
|
|
2541
|
-
opts._selectedLogitTokenIds = selectedTokenIds;
|
|
2542
|
-
try {
|
|
2543
|
-
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogits');
|
|
2544
|
-
const logitsByTokenId = {};
|
|
2545
|
-
for (let index = 0; index < selectedTokenIds.length; index += 1) {
|
|
2546
|
-
logitsByTokenId[selectedTokenIds[index]] = logits[index];
|
|
2547
|
-
}
|
|
2548
|
-
return {
|
|
2549
|
-
seqLen: this.#state.currentSeqLen,
|
|
2550
|
-
tokens: inputIds,
|
|
2551
|
-
tokenIds: selectedTokenIds,
|
|
2552
|
-
logits,
|
|
2553
|
-
logitsByTokenId,
|
|
2554
|
-
phase: {
|
|
2555
|
-
...(prefillPhase ?? {}),
|
|
2556
|
-
totalMs: performance.now() - totalStart,
|
|
2557
|
-
selectedTokenCount: selectedTokenIds.length,
|
|
2558
|
-
},
|
|
2559
|
-
};
|
|
2560
|
-
} finally {
|
|
2561
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
2562
|
-
}
|
|
563
|
+
return prefillWithTokenLogitsImpl.apply(this, arguments);
|
|
2563
564
|
}
|
|
2564
565
|
|
|
2565
566
|
async prefillWithTokenLogitsFromKV(prefix, prompt, tokenIds, options = {}) {
|
|
2566
|
-
|
|
2567
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2568
|
-
throw new Error('Generation already in progress');
|
|
2569
|
-
}
|
|
2570
|
-
assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogitsFromKV');
|
|
2571
|
-
const totalStart = performance.now();
|
|
2572
|
-
if (!prefix || typeof prefix !== 'object' || !prefix.cache) {
|
|
2573
|
-
throw new Error('prefillWithTokenLogitsFromKV.prefix must be a KV cache snapshot.');
|
|
2574
|
-
}
|
|
2575
|
-
if (!Number.isInteger(prefix.seqLen) || prefix.seqLen < 0) {
|
|
2576
|
-
throw new Error('prefillWithTokenLogitsFromKV.prefix.seqLen must be a non-negative integer.');
|
|
2577
|
-
}
|
|
2578
|
-
if (!Array.isArray(prefix.tokens)) {
|
|
2579
|
-
throw new Error('prefillWithTokenLogitsFromKV.prefix.tokens must be an array.');
|
|
2580
|
-
}
|
|
2581
|
-
const selectedTokenIds = normalizeSelectedLogitTokenIds(
|
|
2582
|
-
tokenIds,
|
|
2583
|
-
this.#state.modelConfig.vocabSize,
|
|
2584
|
-
'prefillWithTokenLogitsFromKV.tokenIds'
|
|
2585
|
-
);
|
|
2586
|
-
|
|
2587
|
-
this.#state.kvCache = prefix.cache.clone();
|
|
2588
|
-
if (this.#state.useGPU && this.#state.kvCache) {
|
|
2589
|
-
const device = getDevice();
|
|
2590
|
-
if (device) {
|
|
2591
|
-
this.#state.kvCache.setGPUContext({ device });
|
|
2592
|
-
}
|
|
2593
|
-
}
|
|
2594
|
-
if (
|
|
2595
|
-
hasLinearAttentionLayers(this.#state.modelConfig.layerTypes)
|
|
2596
|
-
&& prefix.linearAttention == null
|
|
2597
|
-
) {
|
|
2598
|
-
throw new Error(
|
|
2599
|
-
'Prefix snapshot is missing linear_attention recurrent state. ' +
|
|
2600
|
-
'Regenerate the prefix snapshot using the current runtime.'
|
|
2601
|
-
);
|
|
2602
|
-
}
|
|
2603
|
-
this.#state.linearAttentionRuntime = restoreLinearAttentionRuntime(
|
|
2604
|
-
this.#state.linearAttentionRuntime,
|
|
2605
|
-
prefix.linearAttention ?? null
|
|
2606
|
-
);
|
|
2607
|
-
this.#state.currentSeqLen = prefix.seqLen;
|
|
2608
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2609
|
-
this.#state.stats.prefillProfileSteps = [];
|
|
2610
|
-
this.#state.decodeRing?.reset();
|
|
2611
|
-
|
|
2612
|
-
const opts = resolvePrefillOptions(this.#state, options);
|
|
2613
|
-
opts._selectedLogitTokenIds = selectedTokenIds;
|
|
2614
|
-
try {
|
|
2615
|
-
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogitsFromKV');
|
|
2616
|
-
const logitsByTokenId = {};
|
|
2617
|
-
for (let index = 0; index < selectedTokenIds.length; index += 1) {
|
|
2618
|
-
logitsByTokenId[selectedTokenIds[index]] = logits[index];
|
|
2619
|
-
}
|
|
2620
|
-
return {
|
|
2621
|
-
seqLen: this.#state.currentSeqLen,
|
|
2622
|
-
prefixTokens: prefix.tokens,
|
|
2623
|
-
tokens: inputIds,
|
|
2624
|
-
tokenIds: selectedTokenIds,
|
|
2625
|
-
logits,
|
|
2626
|
-
logitsByTokenId,
|
|
2627
|
-
phase: {
|
|
2628
|
-
...(prefillPhase ?? {}),
|
|
2629
|
-
totalMs: performance.now() - totalStart,
|
|
2630
|
-
selectedTokenCount: selectedTokenIds.length,
|
|
2631
|
-
prefixTokens: prefix.tokens.length,
|
|
2632
|
-
},
|
|
2633
|
-
};
|
|
2634
|
-
} finally {
|
|
2635
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
2636
|
-
}
|
|
567
|
+
return prefillWithTokenLogitsFromKVImpl.apply(this, arguments);
|
|
2637
568
|
}
|
|
2638
569
|
|
|
2639
|
-
|
|
2640
570
|
async *generateWithPrefixKV(prefix, prompt, options = {}) {
|
|
2641
|
-
|
|
2642
|
-
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
2643
|
-
assertIncrementalDecodeSupport(this.#state, 'generateWithPrefixKV');
|
|
2644
|
-
|
|
2645
|
-
validateCallTimeOptions(options);
|
|
2646
|
-
|
|
2647
|
-
// Apply snapshot
|
|
2648
|
-
this.#state.kvCache = prefix.cache.clone();
|
|
2649
|
-
if (this.#state.useGPU && this.#state.kvCache) {
|
|
2650
|
-
const device = getDevice();
|
|
2651
|
-
if (device) {
|
|
2652
|
-
this.#state.kvCache.setGPUContext({ device });
|
|
2653
|
-
}
|
|
2654
|
-
}
|
|
2655
|
-
if (
|
|
2656
|
-
hasLinearAttentionLayers(this.#state.modelConfig.layerTypes)
|
|
2657
|
-
&& prefix.linearAttention == null
|
|
2658
|
-
) {
|
|
2659
|
-
throw new Error(
|
|
2660
|
-
'Prefix snapshot is missing linear_attention recurrent state. ' +
|
|
2661
|
-
'Regenerate the prefix snapshot using the current runtime.'
|
|
2662
|
-
);
|
|
2663
|
-
}
|
|
2664
|
-
this.#state.linearAttentionRuntime = restoreLinearAttentionRuntime(
|
|
2665
|
-
this.#state.linearAttentionRuntime,
|
|
2666
|
-
prefix.linearAttention ?? null
|
|
2667
|
-
);
|
|
2668
|
-
this.#state.currentSeqLen = prefix.seqLen;
|
|
2669
|
-
|
|
2670
|
-
this.#state.isGenerating = true;
|
|
2671
|
-
this.#state.decodeStepCount = 0;
|
|
2672
|
-
resetActiveExecutionPlan(this.#state);
|
|
2673
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2674
|
-
this.#state.stats.gpuTimeDecodeMs = undefined;
|
|
2675
|
-
this.#state.stats.prefillProfileSteps = [];
|
|
2676
|
-
this.#state.decodeRing?.reset();
|
|
2677
|
-
this.#state.stats.decodeRecordMs = 0;
|
|
2678
|
-
this.#state.stats.decodeRecordOps = 0;
|
|
2679
|
-
this.#state.stats.decodeRecordPasses = 0;
|
|
2680
|
-
this.#state.stats.decodeRecordOpLabels = {};
|
|
2681
|
-
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
2682
|
-
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
2683
|
-
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
2684
|
-
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
2685
|
-
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
2686
|
-
this.#state.stats.prefillRecordMs = 0;
|
|
2687
|
-
this.#state.stats.prefillRecordOps = 0;
|
|
2688
|
-
this.#state.stats.prefillRecordPasses = 0;
|
|
2689
|
-
this.#state.stats.prefillRecordOpLabels = {};
|
|
2690
|
-
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
2691
|
-
this.#state.stats.ttftMs = 0;
|
|
2692
|
-
const startTime = performance.now();
|
|
2693
|
-
|
|
2694
|
-
const opts = resolveGenerateOptions(this.#state, options);
|
|
2695
|
-
|
|
2696
|
-
try {
|
|
2697
|
-
const processedPrompt = resolvePromptInput(this.#state, prompt, opts.useChatTemplate, 'generateWithPrefixKV');
|
|
2698
|
-
|
|
2699
|
-
const inputIds = this.#state.tokenizer.encode(processedPrompt);
|
|
2700
|
-
this._assertTokenIdsInRange(inputIds, 'generateWithPrefixKV.encode');
|
|
2701
|
-
const generatedIds = [...prefix.tokens, ...inputIds];
|
|
2702
|
-
const promptTokenCount = generatedIds.length;
|
|
2703
|
-
this.#state.stats.prefillTokens = inputIds.length;
|
|
2704
|
-
|
|
2705
|
-
const prefillStart = performance.now();
|
|
2706
|
-
const prefillLogits = await this._prefill(inputIds, opts);
|
|
2707
|
-
this.#state.stats.prefillTimeMs = performance.now() - prefillStart;
|
|
2708
|
-
|
|
2709
|
-
applyRepetitionPenalty(prefillLogits, generatedIds, opts.repetitionPenalty);
|
|
2710
|
-
const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
|
|
2711
|
-
const firstToken = sample(prefillLogits, {
|
|
2712
|
-
temperature: opts.temperature,
|
|
2713
|
-
topP: opts.topP,
|
|
2714
|
-
topK: opts.topK,
|
|
2715
|
-
padTokenId,
|
|
2716
|
-
seed: opts.seed,
|
|
2717
|
-
suppressTokenIds: opts.suppressTokenIds,
|
|
2718
|
-
});
|
|
2719
|
-
|
|
2720
|
-
const decodeRuntime = {
|
|
2721
|
-
stopTokenIds: this.#state.modelConfig.stopTokenIds,
|
|
2722
|
-
eosToken: this.#state.tokenizer.getSpecialTokens?.()?.eos,
|
|
2723
|
-
stopSequenceStart: promptTokenCount,
|
|
2724
|
-
decodeToken: (tokenId) => this.#state.tokenizer.decode([tokenId], true, false),
|
|
2725
|
-
logBatchPath: false,
|
|
2726
|
-
};
|
|
2727
|
-
generatedIds.push(firstToken);
|
|
2728
|
-
this.#state.stats.ttftMs = performance.now() - startTime;
|
|
2729
|
-
const firstText = resolveTokenText(
|
|
2730
|
-
this.#state.tokenizer,
|
|
2731
|
-
[firstToken],
|
|
2732
|
-
`[${firstToken}]`,
|
|
2733
|
-
(tokens) => this.#state.tokenizer?.decode?.(tokens, true, false),
|
|
2734
|
-
(tokens) => this.#state.tokenizer?.decode?.(tokens, false, false)
|
|
2735
|
-
);
|
|
2736
|
-
yield firstText;
|
|
2737
|
-
if (options.onToken) options.onToken(firstToken, firstText);
|
|
2738
|
-
|
|
2739
|
-
if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
|
|
2740
|
-
this._recordStopReason('stop-token-or-sequence', firstToken);
|
|
2741
|
-
this.#state.stats.decodeTimeMs = 0;
|
|
2742
|
-
this.#state.stats.tokensGenerated = 1;
|
|
2743
|
-
this.#state.stats.decodeTokens = 1;
|
|
2744
|
-
} else {
|
|
2745
|
-
yield* this._runDecodeLoop(generatedIds, opts, options, decodeRuntime);
|
|
2746
|
-
}
|
|
2747
|
-
this.#state.stats.totalTimeMs = performance.now() - startTime;
|
|
2748
|
-
} finally {
|
|
2749
|
-
this._closeFinitenessFallbackWindow(opts);
|
|
2750
|
-
resetActiveExecutionPlan(this.#state);
|
|
2751
|
-
this.#state.isGenerating = false;
|
|
2752
|
-
}
|
|
571
|
+
return yield* generateWithPrefixKVImpl.apply(this, arguments);
|
|
2753
572
|
}
|
|
2754
573
|
|
|
2755
574
|
// ==========================================================================
|
|
@@ -2757,1226 +576,34 @@ export class PipelineGenerator {
|
|
|
2757
576
|
// ==========================================================================
|
|
2758
577
|
|
|
2759
578
|
async *_runDecodeLoop(generatedIds, opts, options, runtime) {
|
|
2760
|
-
|
|
2761
|
-
stopTokenIds,
|
|
2762
|
-
eosToken,
|
|
2763
|
-
stopSequenceStart,
|
|
2764
|
-
decodeToken,
|
|
2765
|
-
logBatchPath = false,
|
|
2766
|
-
emitMode = 'text',
|
|
2767
|
-
} = runtime;
|
|
2768
|
-
|
|
2769
|
-
let tokensGenerated = 1;
|
|
2770
|
-
markKernelCacheWarmed();
|
|
2771
|
-
|
|
2772
|
-
// Step 4: Lazily initialise PLE buffer cache for decode-path slice reuse.
|
|
2773
|
-
const pleHiddenSize = Number(this.#state.modelConfig.hiddenSizePerLayerInput ?? 0);
|
|
2774
|
-
if (pleHiddenSize > 0 && !this.#state.pleCache) {
|
|
2775
|
-
const activationDtype = resolveActiveExecutionPlan(this.#state).activationDtype;
|
|
2776
|
-
const bpe = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
|
|
2777
|
-
this.#state.pleCache = createPleBufferCache(
|
|
2778
|
-
this.#state.modelConfig.numLayers,
|
|
2779
|
-
pleHiddenSize * bpe,
|
|
2780
|
-
);
|
|
2781
|
-
}
|
|
2782
|
-
if (pleHiddenSize > 0) {
|
|
2783
|
-
await primePleDecodeRuntimeCache(this.#state, generatedIds);
|
|
2784
|
-
}
|
|
2785
|
-
|
|
2786
|
-
const decodeStart = performance.now();
|
|
2787
|
-
const resolvedPerLayerInputsSession = resolvePerLayerInputsSession(
|
|
2788
|
-
this.#state.modelConfig.perLayerInputsSession ?? null,
|
|
2789
|
-
this.#state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
2790
|
-
);
|
|
2791
|
-
const lmHead = this.#state.weights.get('lm_head');
|
|
2792
|
-
const embedBuffer = this.#state.weights.get('embed');
|
|
2793
|
-
const hasCpuWeights = isCpuWeightBuffer(lmHead)
|
|
2794
|
-
|| isCpuWeightBuffer(embedBuffer)
|
|
2795
|
-
|| lmHead instanceof Float32Array
|
|
2796
|
-
|| embedBuffer instanceof Float32Array;
|
|
2797
|
-
const hasLinearLayers = hasLinearAttentionLayers(this.#state.modelConfig.layerTypes);
|
|
2798
|
-
const replayPrefillDecode = usesReplayPrefillDecode(this.#state);
|
|
2799
|
-
const gpuSamplingAvailable = isGPUSamplingAvailable() && !hasCpuWeights;
|
|
2800
|
-
const hasSuppressedSamplingTokens = Array.isArray(opts.suppressTokenIds) && opts.suppressTokenIds.length > 0;
|
|
2801
|
-
const hasRangeBackedPerLayerInputs = hasRangeBackedPerLayerInputEmbeddings({
|
|
2802
|
-
config: this.#state.modelConfig,
|
|
2803
|
-
weights: this.#state.weights,
|
|
2804
|
-
});
|
|
2805
|
-
const hasGpuSplitPerLayerInputs = hasGpuSplitPerLayerInputEmbeddings({
|
|
2806
|
-
config: this.#state.modelConfig,
|
|
2807
|
-
weights: this.#state.weights,
|
|
2808
|
-
perLayerInputsSession: resolvedPerLayerInputsSession,
|
|
2809
|
-
});
|
|
2810
|
-
const pleHotVocabularyRuntime = getPleHotVocabularyRuntime({ weights: this.#state.weights });
|
|
2811
|
-
const resolveCurrentHotVocabularyBatchDecodeAvailable = () => resolveHotVocabularyBatchDecodeAvailability({
|
|
2812
|
-
hasRangeBackedPerLayerInputs,
|
|
2813
|
-
pleHotVocabularyRuntime,
|
|
2814
|
-
tokenId: generatedIds[generatedIds.length - 1] ?? null,
|
|
2815
|
-
});
|
|
2816
|
-
const initialHotVocabularyBatchDecodeAvailable = resolveCurrentHotVocabularyBatchDecodeAvailable();
|
|
2817
|
-
const executionPlan = opts.executionPlan;
|
|
2818
|
-
let useBatchPath = replayPrefillDecode
|
|
2819
|
-
? false
|
|
2820
|
-
: shouldUseBatchDecode({
|
|
2821
|
-
batchSize: executionPlan.batchSize,
|
|
2822
|
-
useGPU: this.#state.useGPU,
|
|
2823
|
-
gpuSamplingAvailable,
|
|
2824
|
-
disableMultiTokenDecode: executionPlan.disableMultiTokenDecode || hasSuppressedSamplingTokens,
|
|
2825
|
-
disableCommandBatching: executionPlan.disableCommandBatching,
|
|
2826
|
-
isBdpaPagedLayout: this.#state.kvCache?.layout === 'bdpa_paged',
|
|
2827
|
-
finitenessFallbackWindowOpen: this._hasFinitenessFallbackWindow(),
|
|
2828
|
-
hasLinearAttentionLayers: hasLinearLayers,
|
|
2829
|
-
selfSpeculationEnabled: opts.speculation?.mode === 'self' && !initialHotVocabularyBatchDecodeAvailable,
|
|
2830
|
-
hasRangeBackedPerLayerInputs,
|
|
2831
|
-
});
|
|
2832
|
-
if (!useBatchPath) {
|
|
2833
|
-
let reason = null;
|
|
2834
|
-
if (replayPrefillDecode) reason = 'replay_prefill_decode';
|
|
2835
|
-
else if (hasCpuWeights) reason = 'cpu_weights';
|
|
2836
|
-
else if (!this.#state.useGPU) reason = 'no_gpu';
|
|
2837
|
-
else if (!gpuSamplingAvailable) reason = 'no_gpu_sampling';
|
|
2838
|
-
else if (executionPlan.disableCommandBatching) reason = 'command_batching_disabled';
|
|
2839
|
-
else if (hasSuppressedSamplingTokens) reason = 'sampling_suppression_requires_cpu_logits';
|
|
2840
|
-
else if (executionPlan.disableMultiTokenDecode) reason = 'multi_token_decode_disabled';
|
|
2841
|
-
else if (executionPlan.batchSize <= 1) reason = 'batch_size_1';
|
|
2842
|
-
else if (this.#state.kvCache?.layout === 'bdpa_paged') reason = 'bdpa_paged_layout';
|
|
2843
|
-
else if (this._hasFinitenessFallbackWindow()) reason = 'finiteness_fallback_window';
|
|
2844
|
-
else if (hasLinearLayers) reason = 'linear_attention_layers';
|
|
2845
|
-
this.#state.stats.decodeMode = replayPrefillDecode
|
|
2846
|
-
? 'replay_prefill'
|
|
2847
|
-
: (opts.speculation?.mode === 'self' ? 'self_speculation' : 'single_token');
|
|
2848
|
-
this.#state.stats.batchGuardReason = reason;
|
|
2849
|
-
} else {
|
|
2850
|
-
this.#state.stats.decodeMode = hasRangeBackedPerLayerInputs
|
|
2851
|
-
? 'batched_gpu_stepwise_ple'
|
|
2852
|
-
: 'batched_gpu';
|
|
2853
|
-
this.#state.stats.batchGuardReason = null;
|
|
2854
|
-
}
|
|
2855
|
-
|
|
2856
|
-
const readbackInterval = executionPlan.readbackInterval;
|
|
2857
|
-
const intervalBatches = readbackInterval == null ? 1 : readbackInterval;
|
|
2858
|
-
const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
|
|
2859
|
-
|
|
2860
|
-
const decodeSingleTokenViaLogits = async () => this._decodeNextTokenViaLogits(generatedIds, opts);
|
|
2861
|
-
|
|
2862
|
-
if (logBatchPath && useBatchPath) {
|
|
2863
|
-
log.debug(
|
|
2864
|
-
'Pipeline',
|
|
2865
|
-
`Using batch decode path with batchSize=${executionPlan.batchSize}, stopCheckMode=${executionPlan.stopCheckMode}, readbackInterval=${readbackInterval}`
|
|
2866
|
-
);
|
|
2867
|
-
}
|
|
2868
|
-
|
|
2869
|
-
while (tokensGenerated < opts.maxTokens) {
|
|
2870
|
-
if (options.signal?.aborted) {
|
|
2871
|
-
this._recordStopReason('aborted');
|
|
2872
|
-
break;
|
|
2873
|
-
}
|
|
2874
|
-
if (this._hasFinitenessFallbackWindow() && useBatchPath) {
|
|
2875
|
-
useBatchPath = false;
|
|
2876
|
-
}
|
|
2877
|
-
const hotVocabularyBatchDecodeAvailable = resolveCurrentHotVocabularyBatchDecodeAvailable();
|
|
2878
|
-
|
|
2879
|
-
if (useBatchPath) {
|
|
2880
|
-
const remaining = opts.maxTokens - tokensGenerated;
|
|
2881
|
-
const maxBatchDecodeTokens = resolveMaxBatchDecodeTokens({
|
|
2882
|
-
hasHotVocabularyBatchDecode: hotVocabularyBatchDecodeAvailable,
|
|
2883
|
-
hasGpuSplitPerLayerInputs,
|
|
2884
|
-
hasLinearAttentionLayers: hasLinearLayers,
|
|
2885
|
-
modelId: this.#state.modelConfig.modelId ?? executionPlan.kernelPathId,
|
|
2886
|
-
activationDtype: executionPlan.activationDtype,
|
|
2887
|
-
currentSeqLen: this.#state.currentSeqLen,
|
|
2888
|
-
maxDecodeTokens: opts.maxTokens,
|
|
2889
|
-
numLayers: this.#state.modelConfig.numLayers,
|
|
2890
|
-
hiddenSize: this.#state.modelConfig.hiddenSize,
|
|
2891
|
-
configuredMaxBatchDecodeTokens: executionPlan.maxBatchDecodeTokens,
|
|
2892
|
-
});
|
|
2893
|
-
const requestedBatchTokens = executionPlan.batchSize * intervalBatches;
|
|
2894
|
-
const boundedBatchTokens = maxBatchDecodeTokens == null
|
|
2895
|
-
? requestedBatchTokens
|
|
2896
|
-
: Math.min(requestedBatchTokens, maxBatchDecodeTokens);
|
|
2897
|
-
const thisBatchSize = Math.min(boundedBatchTokens, remaining);
|
|
2898
|
-
this.#state.batchingStats.requestedBatchTokens = Math.max(
|
|
2899
|
-
this.#state.batchingStats.requestedBatchTokens ?? 0,
|
|
2900
|
-
requestedBatchTokens
|
|
2901
|
-
);
|
|
2902
|
-
this.#state.batchingStats.effectiveBatchTokens = Math.max(
|
|
2903
|
-
this.#state.batchingStats.effectiveBatchTokens ?? 0,
|
|
2904
|
-
thisBatchSize
|
|
2905
|
-
);
|
|
2906
|
-
if (maxBatchDecodeTokens != null) {
|
|
2907
|
-
this.#state.batchingStats.maxBatchTokenCap = Math.max(
|
|
2908
|
-
this.#state.batchingStats.maxBatchTokenCap ?? 0,
|
|
2909
|
-
maxBatchDecodeTokens
|
|
2910
|
-
);
|
|
2911
|
-
}
|
|
2912
|
-
if (thisBatchSize < requestedBatchTokens) {
|
|
2913
|
-
this.#state.batchingStats.batchClampCount = (this.#state.batchingStats.batchClampCount ?? 0) + 1;
|
|
2914
|
-
}
|
|
2915
|
-
const lastToken = generatedIds[generatedIds.length - 1];
|
|
2916
|
-
const boundedExecutionPlan = thisBatchSize < requestedBatchTokens
|
|
2917
|
-
? {
|
|
2918
|
-
...executionPlan,
|
|
2919
|
-
batchSize: Math.min(executionPlan.batchSize, thisBatchSize),
|
|
2920
|
-
readbackInterval: readbackInterval == null
|
|
2921
|
-
? null
|
|
2922
|
-
: Math.min(intervalBatches, thisBatchSize),
|
|
2923
|
-
}
|
|
2924
|
-
: executionPlan;
|
|
2925
|
-
const batchOpts = boundedExecutionPlan === executionPlan
|
|
2926
|
-
? opts
|
|
2927
|
-
: { ...opts, executionPlan: boundedExecutionPlan };
|
|
2928
|
-
|
|
2929
|
-
try {
|
|
2930
|
-
const batchResult = await this._generateNTokensGPU(lastToken, thisBatchSize, generatedIds, batchOpts);
|
|
2931
|
-
let batchTokens = [];
|
|
2932
|
-
let hitStop = false;
|
|
2933
|
-
let stopTokenId = null;
|
|
2934
|
-
for (const tokenId of batchResult.tokens) {
|
|
2935
|
-
if (isStopToken(tokenId, stopTokenIds, eosToken)) {
|
|
2936
|
-
hitStop = true;
|
|
2937
|
-
stopTokenId = tokenId;
|
|
2938
|
-
break;
|
|
2939
|
-
}
|
|
2940
|
-
generatedIds.push(tokenId);
|
|
2941
|
-
tokensGenerated++;
|
|
2942
|
-
if (emitMode === 'token') {
|
|
2943
|
-
yield tokenId;
|
|
2944
|
-
if (options.onToken) options.onToken(tokenId, '');
|
|
2945
|
-
batchTokens.push({ id: tokenId, text: '' });
|
|
2946
|
-
} else {
|
|
2947
|
-
const tokenText = decodeToken(tokenId);
|
|
2948
|
-
yield tokenText;
|
|
2949
|
-
if (options.onToken) options.onToken(tokenId, tokenText);
|
|
2950
|
-
batchTokens.push({ id: tokenId, text: tokenText });
|
|
2951
|
-
}
|
|
2952
|
-
if (batchTokens.length === executionPlan.batchSize) {
|
|
2953
|
-
if (options.onBatch) options.onBatch(batchTokens);
|
|
2954
|
-
batchTokens = [];
|
|
2955
|
-
}
|
|
2956
|
-
}
|
|
2957
|
-
if (batchTokens.length > 0 && options.onBatch) options.onBatch(batchTokens);
|
|
2958
|
-
if (opts.stopSequences.length > 0) {
|
|
2959
|
-
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
2960
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
2961
|
-
this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
|
|
2962
|
-
break;
|
|
2963
|
-
}
|
|
2964
|
-
}
|
|
2965
|
-
if (hitStop) {
|
|
2966
|
-
this._recordStopReason('stop-token', stopTokenId);
|
|
2967
|
-
break;
|
|
2968
|
-
}
|
|
2969
|
-
if (shouldDisableBatchDecodeAfterShortBatch({
|
|
2970
|
-
hitStop,
|
|
2971
|
-
actualCount: batchResult.actualCount,
|
|
2972
|
-
requestedCount: thisBatchSize,
|
|
2973
|
-
})) {
|
|
2974
|
-
useBatchPath = false;
|
|
2975
|
-
continue;
|
|
2976
|
-
}
|
|
2977
|
-
} catch (error) {
|
|
2978
|
-
log.warn('Pipeline', `Batch decode failed, falling back to single-token: ${error}`);
|
|
2979
|
-
useBatchPath = false;
|
|
2980
|
-
let nextToken;
|
|
2981
|
-
try {
|
|
2982
|
-
nextToken = await decodeSingleTokenViaLogits();
|
|
2983
|
-
} catch (singleTokenError) {
|
|
2984
|
-
if (this._shouldUseFinitenessFallback(singleTokenError, `decode-batch-step-${tokensGenerated}`)) {
|
|
2985
|
-
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at batch step ${tokensGenerated}. Truncating KV cache and retrying token with F32 precision.`);
|
|
2986
|
-
nextToken = await this._retryDecodeStepWithFinitenessWindow(
|
|
2987
|
-
generatedIds,
|
|
2988
|
-
opts,
|
|
2989
|
-
`decode-batch-step-${tokensGenerated}`
|
|
2990
|
-
);
|
|
2991
|
-
} else {
|
|
2992
|
-
throw singleTokenError;
|
|
2993
|
-
}
|
|
2994
|
-
}
|
|
2995
|
-
generatedIds.push(nextToken);
|
|
2996
|
-
tokensGenerated++;
|
|
2997
|
-
if (emitMode === 'token') {
|
|
2998
|
-
yield nextToken;
|
|
2999
|
-
if (options.onToken) options.onToken(nextToken, '');
|
|
3000
|
-
} else {
|
|
3001
|
-
const tokenText = decodeToken(nextToken);
|
|
3002
|
-
yield tokenText;
|
|
3003
|
-
if (options.onToken) options.onToken(nextToken, tokenText);
|
|
3004
|
-
}
|
|
3005
|
-
this._consumeFinitenessFallbackToken(opts);
|
|
3006
|
-
if (isStopToken(nextToken, stopTokenIds, eosToken)) {
|
|
3007
|
-
this._recordStopReason('stop-token', nextToken);
|
|
3008
|
-
break;
|
|
3009
|
-
}
|
|
3010
|
-
}
|
|
3011
|
-
} else if (opts.speculation?.mode === 'self') {
|
|
3012
|
-
// Self-speculation: decode one base token plus a configurable burst of
|
|
3013
|
-
// speculative tokens per iteration. Same-model speculation always
|
|
3014
|
-
// accepts under greedy because the model is deterministic — both base
|
|
3015
|
-
// and speculative use the same weights and state. The benefit is
|
|
3016
|
-
// amortizing per-iteration overhead for models where batch decode is
|
|
3017
|
-
// disabled (e.g., linear attention).
|
|
3018
|
-
const speculativeBurstTokens = opts.speculation.tokens;
|
|
3019
|
-
if (!Number.isInteger(speculativeBurstTokens) || speculativeBurstTokens < 1) {
|
|
3020
|
-
throw new Error('[Pipeline] resolved self-speculation tokens must be a positive integer.');
|
|
3021
|
-
}
|
|
3022
|
-
const doSpecDecode = hasLinearLayers
|
|
3023
|
-
? () => this._decodeStep(generatedIds, opts)
|
|
3024
|
-
: decodeSingleTokenViaLogits;
|
|
3025
|
-
|
|
3026
|
-
// Base decode
|
|
3027
|
-
let baseToken;
|
|
3028
|
-
try {
|
|
3029
|
-
baseToken = await doSpecDecode();
|
|
3030
|
-
} catch (error) {
|
|
3031
|
-
if (this._shouldUseFinitenessFallback(error, `spec-base-${tokensGenerated}`)) {
|
|
3032
|
-
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated} (speculation:base). Retrying.`);
|
|
3033
|
-
baseToken = await this._retryDecodeStepWithFinitenessWindow(generatedIds, opts, `spec-base-${tokensGenerated}`);
|
|
3034
|
-
} else {
|
|
3035
|
-
throw error;
|
|
3036
|
-
}
|
|
3037
|
-
}
|
|
3038
|
-
generatedIds.push(baseToken);
|
|
3039
|
-
tokensGenerated++;
|
|
3040
|
-
if (emitMode === 'token') {
|
|
3041
|
-
yield baseToken;
|
|
3042
|
-
if (options.onToken) options.onToken(baseToken, '');
|
|
3043
|
-
} else {
|
|
3044
|
-
const text = decodeToken(baseToken);
|
|
3045
|
-
yield text;
|
|
3046
|
-
if (options.onToken) options.onToken(baseToken, text);
|
|
3047
|
-
}
|
|
3048
|
-
this._consumeFinitenessFallbackToken(opts);
|
|
3049
|
-
|
|
3050
|
-
if (isStopToken(baseToken, stopTokenIds, eosToken)) {
|
|
3051
|
-
this._recordStopReason('stop-token', baseToken);
|
|
3052
|
-
break;
|
|
3053
|
-
}
|
|
3054
|
-
if (tokensGenerated >= opts.maxTokens) break;
|
|
3055
|
-
if (opts.stopSequences.length > 0) {
|
|
3056
|
-
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
3057
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
3058
|
-
this._recordStopReason('stop-sequence', baseToken);
|
|
3059
|
-
break;
|
|
3060
|
-
}
|
|
3061
|
-
}
|
|
3062
|
-
|
|
3063
|
-
for (let specIndex = 0; specIndex < speculativeBurstTokens; specIndex += 1) {
|
|
3064
|
-
if (tokensGenerated >= opts.maxTokens) {
|
|
3065
|
-
break;
|
|
3066
|
-
}
|
|
3067
|
-
let specToken;
|
|
3068
|
-
try {
|
|
3069
|
-
specToken = await doSpecDecode();
|
|
3070
|
-
} catch (error) {
|
|
3071
|
-
if (this._shouldUseFinitenessFallback(error, `spec-extra-${tokensGenerated}`)) {
|
|
3072
|
-
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated} (speculation:spec). Retrying.`);
|
|
3073
|
-
specToken = await this._retryDecodeStepWithFinitenessWindow(generatedIds, opts, `spec-extra-${tokensGenerated}`);
|
|
3074
|
-
} else {
|
|
3075
|
-
throw error;
|
|
3076
|
-
}
|
|
3077
|
-
}
|
|
3078
|
-
generatedIds.push(specToken);
|
|
3079
|
-
tokensGenerated++;
|
|
3080
|
-
this.#state.stats.speculationAttempts = (this.#state.stats.speculationAttempts ?? 0) + 1;
|
|
3081
|
-
this.#state.stats.speculationAccepted = (this.#state.stats.speculationAccepted ?? 0) + 1;
|
|
3082
|
-
if (emitMode === 'token') {
|
|
3083
|
-
yield specToken;
|
|
3084
|
-
if (options.onToken) options.onToken(specToken, '');
|
|
3085
|
-
} else {
|
|
3086
|
-
const text = decodeToken(specToken);
|
|
3087
|
-
yield text;
|
|
3088
|
-
if (options.onToken) options.onToken(specToken, text);
|
|
3089
|
-
}
|
|
3090
|
-
this._consumeFinitenessFallbackToken(opts);
|
|
3091
|
-
|
|
3092
|
-
if (opts.debug || opts.benchmark) {
|
|
3093
|
-
const elapsedMs = performance.now() - decodeStart;
|
|
3094
|
-
const tokPerSec = (tokensGenerated / elapsedMs) * 1000;
|
|
3095
|
-
log.debug('Decode', `#${tokensGenerated} speculation:self (${tokPerSec.toFixed(2)} tok/s avg)`);
|
|
3096
|
-
}
|
|
3097
|
-
|
|
3098
|
-
if (isStopToken(specToken, stopTokenIds, eosToken)) {
|
|
3099
|
-
this._recordStopReason('stop-token', specToken);
|
|
3100
|
-
break;
|
|
3101
|
-
}
|
|
3102
|
-
if (opts.stopSequences.length > 0) {
|
|
3103
|
-
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
3104
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
3105
|
-
this._recordStopReason('stop-sequence', specToken);
|
|
3106
|
-
break;
|
|
3107
|
-
}
|
|
3108
|
-
}
|
|
3109
|
-
}
|
|
3110
|
-
if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken)) {
|
|
3111
|
-
this._recordStopReason('stop-token', generatedIds[generatedIds.length - 1]);
|
|
3112
|
-
break;
|
|
3113
|
-
}
|
|
3114
|
-
if (opts.stopSequences.length > 0) {
|
|
3115
|
-
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
3116
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
3117
|
-
this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
|
|
3118
|
-
break;
|
|
3119
|
-
}
|
|
3120
|
-
}
|
|
3121
|
-
} else {
|
|
3122
|
-
const tokenStart = performance.now();
|
|
3123
|
-
let nextToken;
|
|
3124
|
-
try {
|
|
3125
|
-
nextToken = hasLinearLayers
|
|
3126
|
-
? await this._decodeStep(generatedIds, opts)
|
|
3127
|
-
: await decodeSingleTokenViaLogits();
|
|
3128
|
-
} catch (error) {
|
|
3129
|
-
if (this._shouldUseFinitenessFallback(error, `decode-step-${tokensGenerated}`)) {
|
|
3130
|
-
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated}. Truncating KV cache and retrying token with F32 precision.`);
|
|
3131
|
-
nextToken = await this._retryDecodeStepWithFinitenessWindow(
|
|
3132
|
-
generatedIds,
|
|
3133
|
-
opts,
|
|
3134
|
-
`decode-step-${tokensGenerated}`
|
|
3135
|
-
);
|
|
3136
|
-
} else {
|
|
3137
|
-
throw error;
|
|
3138
|
-
}
|
|
3139
|
-
}
|
|
3140
|
-
const tokenTime = performance.now() - tokenStart;
|
|
3141
|
-
generatedIds.push(nextToken);
|
|
3142
|
-
tokensGenerated++;
|
|
3143
|
-
|
|
3144
|
-
// Step 5: Fire-and-forget prefetch of next token's PLE row.
|
|
3145
|
-
if (pleHiddenSize > 0) {
|
|
3146
|
-
const pleWeights = this.#state.weights.get('per_layer_inputs');
|
|
3147
|
-
if (pleWeights?.embedTokensPerLayer) {
|
|
3148
|
-
this.#state.plePrefetchPending = prefetchPerLayerRow(
|
|
3149
|
-
nextToken,
|
|
3150
|
-
pleWeights.embedTokensPerLayer,
|
|
3151
|
-
this.#state.modelConfig.numLayers * pleHiddenSize,
|
|
3152
|
-
resolvedPerLayerInputsSession,
|
|
3153
|
-
);
|
|
3154
|
-
}
|
|
3155
|
-
}
|
|
3156
|
-
|
|
3157
|
-
const tokenText = emitMode === 'token' ? '' : decodeToken(nextToken);
|
|
3158
|
-
if (emitMode === 'token') {
|
|
3159
|
-
yield nextToken;
|
|
3160
|
-
if (options.onToken) options.onToken(nextToken, '');
|
|
3161
|
-
} else {
|
|
3162
|
-
yield tokenText;
|
|
3163
|
-
if (options.onToken) options.onToken(nextToken, tokenText);
|
|
3164
|
-
}
|
|
3165
|
-
this._consumeFinitenessFallbackToken(opts);
|
|
3166
|
-
|
|
3167
|
-
if (opts.debug || opts.benchmark) {
|
|
3168
|
-
const elapsedMs = performance.now() - decodeStart;
|
|
3169
|
-
const tokPerSec = (tokensGenerated / elapsedMs) * 1000;
|
|
3170
|
-
log.debug('Decode', `#${tokensGenerated} "${tokenText}" ${tokenTime.toFixed(0)}ms (${tokPerSec.toFixed(2)} tok/s avg)`);
|
|
3171
|
-
}
|
|
3172
|
-
|
|
3173
|
-
if (isStopToken(nextToken, stopTokenIds, eosToken)) {
|
|
3174
|
-
this._recordStopReason('stop-token', nextToken);
|
|
3175
|
-
break;
|
|
3176
|
-
}
|
|
3177
|
-
if (opts.stopSequences.length > 0) {
|
|
3178
|
-
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
3179
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
3180
|
-
this._recordStopReason('stop-sequence', nextToken);
|
|
3181
|
-
break;
|
|
3182
|
-
}
|
|
3183
|
-
}
|
|
3184
|
-
}
|
|
3185
|
-
}
|
|
3186
|
-
|
|
3187
|
-
if (!this.#state.stats.stopReason) {
|
|
3188
|
-
this._recordStopReason(tokensGenerated >= opts.maxTokens ? 'max-tokens' : 'completed');
|
|
3189
|
-
}
|
|
3190
|
-
this.#state.stats.decodeTimeMs = performance.now() - decodeStart;
|
|
3191
|
-
this.#state.stats.tokensGenerated = tokensGenerated;
|
|
3192
|
-
this.#state.stats.decodeTokens = tokensGenerated;
|
|
3193
|
-
this.#state.stats.batching = { ...this.#state.batchingStats };
|
|
579
|
+
return yield* _runDecodeLoopImpl.apply(this, arguments);
|
|
3194
580
|
}
|
|
3195
581
|
|
|
3196
582
|
async _prefillToHidden(inputIds, opts) {
|
|
3197
|
-
|
|
3198
|
-
return this._prefill(inputIds, { ...opts, _returnHidden: true });
|
|
583
|
+
return _prefillToHiddenImpl.apply(this, arguments);
|
|
3199
584
|
}
|
|
3200
585
|
|
|
3201
|
-
|
|
3202
586
|
async _prefill(inputIds, opts) {
|
|
3203
|
-
|
|
3204
|
-
const config = this.#state.modelConfig;
|
|
3205
|
-
const selectedLogitTokenIds = normalizeSelectedLogitTokenIds(
|
|
3206
|
-
opts?._selectedLogitTokenIds,
|
|
3207
|
-
config.vocabSize,
|
|
3208
|
-
'_prefill.selectedLogitTokenIds'
|
|
3209
|
-
);
|
|
3210
|
-
const startPos = this.#state.currentSeqLen;
|
|
3211
|
-
const tracePrefillEnabled = isTraceEnabled('perf');
|
|
3212
|
-
const prefillTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
3213
|
-
const returnHidden = opts?._returnHidden === true;
|
|
3214
|
-
const embeddingInputIds = resolvePrefillEmbeddingInputIds(
|
|
3215
|
-
inputIds,
|
|
3216
|
-
opts?.embeddingInputSpan ?? null,
|
|
3217
|
-
'_prefill'
|
|
3218
|
-
);
|
|
3219
|
-
const multimodalBidirectionalSpan = resolvePrefillMultimodalBidirectionalSpan(
|
|
3220
|
-
inputIds,
|
|
3221
|
-
opts?.multimodalBidirectionalSpan ?? null,
|
|
3222
|
-
'_prefill'
|
|
3223
|
-
);
|
|
3224
|
-
if (embeddingInputIds !== inputIds) {
|
|
3225
|
-
this._assertTokenIdsInRange(embeddingInputIds, '_prefill.embeddingInputIds');
|
|
3226
|
-
}
|
|
3227
|
-
const embeddingOverride = normalizePrefixEmbeddingOverride(
|
|
3228
|
-
opts?.embeddingOverrides ?? null,
|
|
3229
|
-
config.hiddenSize,
|
|
3230
|
-
numTokens,
|
|
3231
|
-
'_prefill'
|
|
3232
|
-
);
|
|
3233
|
-
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
3234
|
-
this.#state.stats.prefillProfileSteps = [];
|
|
3235
|
-
|
|
3236
|
-
if (startPos === 0 && hasLinearAttentionLayers(config.layerTypes)) {
|
|
3237
|
-
this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
|
|
3238
|
-
}
|
|
3239
|
-
if (startPos === 0) {
|
|
3240
|
-
for (const [, convState] of this.#state.convLayerStates) {
|
|
3241
|
-
if (convState.convStateGPU && convState.hiddenSize && convState.kernelSize) {
|
|
3242
|
-
uploadData(convState.convStateGPU, new Float32Array(convState.hiddenSize * (convState.kernelSize - 1)));
|
|
3243
|
-
}
|
|
3244
|
-
}
|
|
3245
|
-
}
|
|
3246
|
-
|
|
3247
|
-
const embedBufferRaw = this.#state.weights.get('embed');
|
|
3248
|
-
if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
|
|
3249
|
-
throw new Error('Embed buffer not found or not a supported buffer type');
|
|
3250
|
-
}
|
|
3251
|
-
const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
|
|
3252
|
-
const embedDtype = isCpuWeightBuffer(embedBufferRaw)
|
|
3253
|
-
? embedBufferRaw.dtype
|
|
3254
|
-
: getWeightDtype(embedBufferRaw);
|
|
3255
|
-
const embedMetadata = getWeightMetadata(embedBufferRaw);
|
|
3256
|
-
if (opts.debug) {
|
|
3257
|
-
const embedSize = isGpuBufferInstance(embedBuffer) ? embedBuffer.size : 'N/A';
|
|
3258
|
-
log.debug('Pipeline', `Embed buffer: type=${embedBuffer?.constructor?.name}, size=${embedSize}, dtype=${embedDtype}`);
|
|
3259
|
-
}
|
|
3260
|
-
|
|
3261
|
-
const device = getDevice();
|
|
3262
|
-
const useCheckpoints = opts.debugLayers && opts.debugLayers.length > 0;
|
|
3263
|
-
const disableCommandBatching = shouldDisablePrefillCommandBatching(
|
|
3264
|
-
this.#state,
|
|
3265
|
-
opts,
|
|
3266
|
-
multimodalBidirectionalSpan
|
|
3267
|
-
);
|
|
3268
|
-
const createRecorder = (label) => {
|
|
3269
|
-
if (!device || disableCommandBatching) return undefined;
|
|
3270
|
-
return opts.profile
|
|
3271
|
-
? createProfilingRecorder(label, device)
|
|
3272
|
-
: createCommandRecorder(label, { recordLabels: opts.debug === true || opts.benchmark === true }, device);
|
|
3273
|
-
};
|
|
3274
|
-
const recorder = createRecorder('prefill');
|
|
3275
|
-
const debugCheckBuffer = this.#state.debug
|
|
3276
|
-
? (buffer, label, numTokens, expectedDim) =>
|
|
3277
|
-
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
3278
|
-
: undefined;
|
|
3279
|
-
const context = buildLayerContext(
|
|
3280
|
-
this.#state,
|
|
3281
|
-
recorder,
|
|
3282
|
-
false,
|
|
3283
|
-
opts.debugLayers,
|
|
3284
|
-
debugCheckBuffer,
|
|
3285
|
-
opts.executionPlan
|
|
3286
|
-
);
|
|
3287
|
-
context.trainingCapture = opts?._trainingCapture ?? null;
|
|
3288
|
-
context.currentTokenIds = inputIds;
|
|
3289
|
-
context.diffusionGemmaDecoder = opts?._diffusionGemmaDecoder === true;
|
|
3290
|
-
context.skipKVCacheWrites = returnHidden
|
|
3291
|
-
&& opts?._embeddingOnly === true
|
|
3292
|
-
&& this.#state.runtimeConfig?.inference?.session?.skipEmbeddingKVCacheWrites === true;
|
|
3293
|
-
context.multimodalBidirectionalSpan = multimodalBidirectionalSpan == null
|
|
3294
|
-
? null
|
|
3295
|
-
: {
|
|
3296
|
-
start: startPos + multimodalBidirectionalSpan.offset,
|
|
3297
|
-
length: multimodalBidirectionalSpan.length,
|
|
3298
|
-
};
|
|
3299
|
-
let gpuTimePrefillMs = 0;
|
|
3300
|
-
let hasGpuTimePrefill = false;
|
|
3301
|
-
const recordProfile = async (rec) => {
|
|
3302
|
-
if (!opts.profile || !rec) return;
|
|
3303
|
-
const timings = rec.isProfilingEnabled()
|
|
3304
|
-
? await rec.resolveProfileTimings()
|
|
3305
|
-
: null;
|
|
3306
|
-
const total = sumProfileTimings(timings);
|
|
3307
|
-
if (total !== null) {
|
|
3308
|
-
gpuTimePrefillMs += total;
|
|
3309
|
-
hasGpuTimePrefill = true;
|
|
3310
|
-
}
|
|
3311
|
-
if (timings || opts.executionObserver) {
|
|
3312
|
-
recordPrefillProfileStep(this.#state, {
|
|
3313
|
-
label: rec.label,
|
|
3314
|
-
timings,
|
|
3315
|
-
totalMs: total ?? undefined,
|
|
3316
|
-
recorderStats: rec.getStats(),
|
|
3317
|
-
});
|
|
3318
|
-
if (!opts.executionObserver) {
|
|
3319
|
-
log.warn('Profile', `Prefill (${rec.label}):`);
|
|
3320
|
-
log.warn('Profile', CommandRecorder.formatProfileReport(timings));
|
|
3321
|
-
}
|
|
3322
|
-
}
|
|
3323
|
-
};
|
|
3324
|
-
|
|
3325
|
-
const benchmarkSubmits = opts.debug;
|
|
3326
|
-
if (benchmarkSubmits) {
|
|
3327
|
-
setTrackSubmits(true);
|
|
3328
|
-
resetSubmitStats();
|
|
3329
|
-
}
|
|
3330
|
-
|
|
3331
|
-
const preserveBufferAcrossRecorderSubmit = (buffer, activeRecorder, label) => {
|
|
3332
|
-
if (!activeRecorder || !isGpuBufferInstance(buffer)) {
|
|
3333
|
-
return buffer;
|
|
3334
|
-
}
|
|
3335
|
-
const carryBuffer = acquireBuffer(
|
|
3336
|
-
buffer.size,
|
|
3337
|
-
typeof buffer.usage === 'number' ? buffer.usage : undefined,
|
|
3338
|
-
label
|
|
3339
|
-
);
|
|
3340
|
-
activeRecorder.getEncoder().copyBufferToBuffer(buffer, 0, carryBuffer, 0, buffer.size);
|
|
3341
|
-
activeRecorder.trackTemporaryBuffer(buffer);
|
|
3342
|
-
return carryBuffer;
|
|
3343
|
-
};
|
|
3344
|
-
|
|
3345
|
-
const activationDtype = opts.executionPlan?.activationDtype ?? this._getEffectiveActivationDtype();
|
|
3346
|
-
const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
|
|
3347
|
-
if (tracePrefillEnabled) {
|
|
3348
|
-
trace.perf('Prefill phase start', {
|
|
3349
|
-
numTokens,
|
|
3350
|
-
startPos,
|
|
3351
|
-
numLayers: config.numLayers,
|
|
3352
|
-
activationDtype,
|
|
3353
|
-
returnHidden,
|
|
3354
|
-
});
|
|
3355
|
-
}
|
|
3356
|
-
const embedTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
3357
|
-
let baseEmbeddings = await embed(embeddingInputIds, embedBuffer, {
|
|
3358
|
-
hiddenSize: config.hiddenSize,
|
|
3359
|
-
vocabSize: config.vocabSize,
|
|
3360
|
-
scaleEmbeddings: config.scaleEmbeddings,
|
|
3361
|
-
embeddingScale: config.embeddingScale,
|
|
3362
|
-
debug: opts.debug,
|
|
3363
|
-
recorder,
|
|
3364
|
-
transpose: this.#state.embeddingTranspose,
|
|
3365
|
-
debugProbes: this.#state.runtimeConfig.shared.debug.probes,
|
|
3366
|
-
operatorDiagnostics: this.#state.operatorDiagnostics,
|
|
3367
|
-
activationDtype,
|
|
3368
|
-
embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
|
|
3369
|
-
embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
|
|
3370
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
3371
|
-
});
|
|
3372
|
-
if (tracePrefillEnabled) {
|
|
3373
|
-
trace.perf('Prefill embed complete', {
|
|
3374
|
-
numTokens,
|
|
3375
|
-
elapsedMs: performance.now() - embedTraceStart,
|
|
3376
|
-
});
|
|
3377
|
-
}
|
|
3378
|
-
let hiddenStates = baseEmbeddings;
|
|
3379
|
-
let perLayerInputs = null;
|
|
3380
|
-
const perLayerInputsTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
3381
|
-
try {
|
|
3382
|
-
hiddenStates = await applyPrefixEmbeddingOverride(
|
|
3383
|
-
baseEmbeddings,
|
|
3384
|
-
embeddingOverride,
|
|
3385
|
-
config.hiddenSize,
|
|
3386
|
-
'_prefill',
|
|
3387
|
-
{
|
|
3388
|
-
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
3389
|
-
transitionDeclaredBy: resolvePrefixEmbeddingOverrideTransitionDeclaredBy(this.#state.executionV1State),
|
|
3390
|
-
}
|
|
3391
|
-
);
|
|
3392
|
-
perLayerInputs = await preparePerLayerInputs(
|
|
3393
|
-
embeddingInputIds,
|
|
3394
|
-
embeddingInputIds === inputIds ? hiddenStates : baseEmbeddings,
|
|
3395
|
-
context,
|
|
3396
|
-
{
|
|
3397
|
-
numTokens,
|
|
3398
|
-
pleCache: this.#state.pleCache ?? null,
|
|
3399
|
-
}
|
|
3400
|
-
);
|
|
3401
|
-
if (tracePrefillEnabled) {
|
|
3402
|
-
trace.perf('Prefill per-layer inputs complete', {
|
|
3403
|
-
numTokens,
|
|
3404
|
-
elapsedMs: performance.now() - perLayerInputsTraceStart,
|
|
3405
|
-
materialized: Array.isArray(perLayerInputs),
|
|
3406
|
-
});
|
|
3407
|
-
}
|
|
3408
|
-
} catch (error) {
|
|
3409
|
-
if (isGpuBufferInstance(hiddenStates?.buffer)) {
|
|
3410
|
-
releaseBuffer(hiddenStates.buffer);
|
|
3411
|
-
}
|
|
3412
|
-
if (hiddenStates === baseEmbeddings) {
|
|
3413
|
-
baseEmbeddings = null;
|
|
3414
|
-
}
|
|
3415
|
-
hiddenStates = null;
|
|
3416
|
-
throw error;
|
|
3417
|
-
} finally {
|
|
3418
|
-
if (hiddenStates !== baseEmbeddings && isGpuBufferInstance(baseEmbeddings?.buffer)) {
|
|
3419
|
-
releaseBuffer(baseEmbeddings.buffer);
|
|
3420
|
-
}
|
|
3421
|
-
baseEmbeddings = null;
|
|
3422
|
-
}
|
|
3423
|
-
|
|
3424
|
-
if (opts.debug && isGpuBufferInstance(hiddenStates)) {
|
|
3425
|
-
if (recorder) {
|
|
3426
|
-
hiddenStates = createTensor(
|
|
3427
|
-
preserveBufferAcrossRecorderSubmit(hiddenStates.buffer, recorder, 'prefill_embed_carry'),
|
|
3428
|
-
hiddenStates.dtype,
|
|
3429
|
-
hiddenStates.shape,
|
|
3430
|
-
hiddenStates.label
|
|
3431
|
-
);
|
|
3432
|
-
recordPrefillRecorderStats(this.#state, recorder);
|
|
3433
|
-
await recorder.submitAndWait();
|
|
3434
|
-
await recordProfile(recorder);
|
|
3435
|
-
}
|
|
3436
|
-
const debugReadbackSize = this.#state.runtimeConfig.shared.debug.pipeline.readbackSampleSize;
|
|
3437
|
-
const sample = await readBuffer(hiddenStates, Math.min(debugReadbackSize, hiddenStates.size));
|
|
3438
|
-
const f32 = decodeReadback(sample, activationDtype);
|
|
3439
|
-
const nanCount = f32.filter(x => !Number.isFinite(x)).length;
|
|
3440
|
-
let maxAbs = 0;
|
|
3441
|
-
for (let i = 0; i < f32.length; i++) {
|
|
3442
|
-
const abs = Math.abs(f32[i]);
|
|
3443
|
-
if (abs > maxAbs) maxAbs = abs;
|
|
3444
|
-
}
|
|
3445
|
-
const first8 = Array.from(f32).slice(0, 8).map(x => x.toFixed(4)).join(', ');
|
|
3446
|
-
log.debug('Pipeline', `After embed: buffer.label=${hiddenStates.label}, buffer.size=${hiddenStates.size}, maxAbs=${maxAbs.toFixed(4)}`);
|
|
3447
|
-
log.debug('Pipeline', `After embed first8=[${first8}], nan=${nanCount}/${f32.length}`);
|
|
3448
|
-
}
|
|
3449
|
-
|
|
3450
|
-
if (opts.debug) {
|
|
3451
|
-
log.debug('Pipeline', `LAYER_LOOP_START: numLayers=${config.numLayers}, useGPU=${context.useGPU}`);
|
|
3452
|
-
}
|
|
3453
|
-
|
|
3454
|
-
if (this.#state.finitenessBuffer) {
|
|
3455
|
-
const device = getDevice();
|
|
3456
|
-
if (device) {
|
|
3457
|
-
device.queue.writeBuffer(this.#state.finitenessBuffer, 0, FINITENESS_RESET_WORDS);
|
|
3458
|
-
}
|
|
3459
|
-
}
|
|
3460
|
-
|
|
3461
|
-
let currentRecorder = recorder;
|
|
3462
|
-
|
|
3463
|
-
// Chunked recorder submission: submit every N layers to release tracked intermediate
|
|
3464
|
-
// buffers, preventing unbounded memory growth during large prefills. Critical for
|
|
3465
|
-
// replay_prefill models where each decode step re-runs a prefill-style layer pass.
|
|
3466
|
-
const prefillRecorderChunkLayers = resolvePrefillRecorderChunkLayers({
|
|
3467
|
-
configuredPrefillChunkLayers: this.#state.runtimeConfig?.inference?.session?.prefillChunkLayers,
|
|
3468
|
-
hasGpuSplitPerLayerInputs: context.perLayerInputsSession?.materialization === 'gpu_split_tables',
|
|
3469
|
-
numTokens,
|
|
3470
|
-
});
|
|
3471
|
-
|
|
3472
|
-
let currentHiddenBuffer = hiddenStates.buffer;
|
|
3473
|
-
let prefillRecordMs = 0;
|
|
3474
|
-
let prefillSubmitWaitMs = 0;
|
|
3475
|
-
const layerLoopTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
3476
|
-
const traceLayerHealthEnabled = isTraceEnabled('logits');
|
|
3477
|
-
try {
|
|
3478
|
-
for (let l = 0; l < config.numLayers; l++) {
|
|
3479
|
-
// Per-layer hard cancellation: when the caller's AbortSignal aborts,
|
|
3480
|
-
// exit the prefill loop between layer dispatches rather than
|
|
3481
|
-
// continuing to burn GPU on superseded work. Granularity is one
|
|
3482
|
-
// layer (~5-50ms), which is the fastest cancel granularity WebGPU
|
|
3483
|
-
// exposes today.
|
|
3484
|
-
if (opts?.signal?.aborted) {
|
|
3485
|
-
const reason = typeof opts.signal.reason === "string" ? opts.signal.reason : "Doppler: prefill aborted";
|
|
3486
|
-
const err = new Error(reason);
|
|
3487
|
-
err.name = "AbortError";
|
|
3488
|
-
err.code = "ABORT_ERR";
|
|
3489
|
-
throw err;
|
|
3490
|
-
}
|
|
3491
|
-
context.recorder = currentRecorder;
|
|
3492
|
-
context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
|
|
3493
|
-
|
|
3494
|
-
const prevBuffer = currentHiddenBuffer;
|
|
3495
|
-
const layerRecordStart = performance.now();
|
|
3496
|
-
const layerOutput = await processLayer(l, currentHiddenBuffer, numTokens, true, context);
|
|
3497
|
-
prefillRecordMs += performance.now() - layerRecordStart;
|
|
3498
|
-
if (!isGpuBufferInstance(layerOutput)) throw new Error('Expected GPUBuffer from processLayer');
|
|
3499
|
-
currentHiddenBuffer = layerOutput;
|
|
3500
|
-
releasePerLayerInputBuffer(
|
|
3501
|
-
context.perLayerInputBuffer,
|
|
3502
|
-
currentRecorder,
|
|
3503
|
-
context.decodeBuffers,
|
|
3504
|
-
this.#state.pleCache ?? null
|
|
3505
|
-
);
|
|
3506
|
-
if (perLayerInputs) {
|
|
3507
|
-
perLayerInputs[l] = null;
|
|
3508
|
-
}
|
|
3509
|
-
context.perLayerInputBuffer = null;
|
|
3510
|
-
|
|
3511
|
-
const isCheckpoint = useCheckpoints && opts.debugLayers?.includes(l);
|
|
3512
|
-
const isChunkBoundary = !isCheckpoint
|
|
3513
|
-
&& !traceLayerHealthEnabled
|
|
3514
|
-
&& currentRecorder
|
|
3515
|
-
&& l < config.numLayers - 1
|
|
3516
|
-
&& (l + 1) % prefillRecorderChunkLayers === 0;
|
|
3517
|
-
|
|
3518
|
-
if (isCheckpoint && currentRecorder) {
|
|
3519
|
-
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3520
|
-
currentHiddenBuffer,
|
|
3521
|
-
currentRecorder,
|
|
3522
|
-
'prefill_checkpoint_carry'
|
|
3523
|
-
);
|
|
3524
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3525
|
-
await currentRecorder.submitAndWait();
|
|
3526
|
-
await recordProfile(currentRecorder);
|
|
3527
|
-
currentRecorder = undefined;
|
|
3528
|
-
}
|
|
3529
|
-
|
|
3530
|
-
if (traceLayerHealthEnabled && currentRecorder) {
|
|
3531
|
-
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3532
|
-
currentHiddenBuffer,
|
|
3533
|
-
currentRecorder,
|
|
3534
|
-
'prefill_trace_layer_health_carry'
|
|
3535
|
-
);
|
|
3536
|
-
const traceSubmitStart = performance.now();
|
|
3537
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3538
|
-
await currentRecorder.submitAndWait();
|
|
3539
|
-
await recordProfile(currentRecorder);
|
|
3540
|
-
prefillSubmitWaitMs += performance.now() - traceSubmitStart;
|
|
3541
|
-
await traceActivationHealth(
|
|
3542
|
-
`PREFILL_LAYER_${l}_HEALTH`,
|
|
3543
|
-
currentHiddenBuffer,
|
|
3544
|
-
activationDtype,
|
|
3545
|
-
numTokens * config.hiddenSize
|
|
3546
|
-
);
|
|
3547
|
-
currentRecorder = l < config.numLayers - 1
|
|
3548
|
-
? createRecorder('prefill-trace')
|
|
3549
|
-
: undefined;
|
|
3550
|
-
}
|
|
3551
|
-
|
|
3552
|
-
const shouldDebug = opts.debug && currentHiddenBuffer && (!recorder || isCheckpoint);
|
|
3553
|
-
if (shouldDebug && !currentRecorder) {
|
|
3554
|
-
const device = getDevice();
|
|
3555
|
-
if (device) {
|
|
3556
|
-
if (allowReadback(`pipeline.prefill.layer-${l}`)) {
|
|
3557
|
-
try {
|
|
3558
|
-
const sampleSize = config.hiddenSize * activationBytes;
|
|
3559
|
-
const lastTokenOffset = (numTokens - 1) * config.hiddenSize * activationBytes;
|
|
3560
|
-
const readback = await readBufferSlice(currentHiddenBuffer, lastTokenOffset, sampleSize);
|
|
3561
|
-
const data = decodeReadback(readback, activationDtype);
|
|
3562
|
-
let min = Infinity;
|
|
3563
|
-
let max = -Infinity;
|
|
3564
|
-
let maxAbs = 0;
|
|
3565
|
-
for (const v of data) {
|
|
3566
|
-
if (!Number.isFinite(v)) continue;
|
|
3567
|
-
if (v < min) min = v;
|
|
3568
|
-
if (v > max) max = v;
|
|
3569
|
-
const av = Math.abs(v);
|
|
3570
|
-
if (av > maxAbs) maxAbs = av;
|
|
3571
|
-
}
|
|
3572
|
-
const sample = Array.from(data).slice(0, 3).map(x => x.toFixed(3)).join(', ');
|
|
3573
|
-
log.debug('Pipeline', `LAYER_${l}_LAST[pos=${numTokens - 1}]: min=${min.toFixed(3)}, max=${max.toFixed(3)}, maxAbs=${maxAbs.toFixed(2)}, sample=[${sample}]`);
|
|
3574
|
-
} catch (e) {
|
|
3575
|
-
log.debug('Pipeline', `LAYER_${l}_LAST: error reading buffer: ${e}`);
|
|
3576
|
-
}
|
|
3577
|
-
}
|
|
3578
|
-
}
|
|
3579
|
-
}
|
|
3580
|
-
|
|
3581
|
-
if (isCheckpoint && useCheckpoints && l < config.numLayers - 1) {
|
|
3582
|
-
currentRecorder = createRecorder('prefill-cont');
|
|
3583
|
-
}
|
|
3584
|
-
|
|
3585
|
-
if (prevBuffer !== currentHiddenBuffer) {
|
|
3586
|
-
if (currentRecorder) {
|
|
3587
|
-
currentRecorder.trackTemporaryBuffer(prevBuffer);
|
|
3588
|
-
} else {
|
|
3589
|
-
releaseBuffer(prevBuffer);
|
|
3590
|
-
}
|
|
3591
|
-
}
|
|
3592
|
-
|
|
3593
|
-
if (isChunkBoundary) {
|
|
3594
|
-
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3595
|
-
currentHiddenBuffer,
|
|
3596
|
-
currentRecorder,
|
|
3597
|
-
'prefill_chunk_carry'
|
|
3598
|
-
);
|
|
3599
|
-
// Chunk boundary exists only to bound intermediate buffer lifetime.
|
|
3600
|
-
// When the runtime opts into async chunk submit AND profile timings
|
|
3601
|
-
// are not being collected, skip the CPU-GPU wait: queue order is
|
|
3602
|
-
// preserved across submits and deferred cleanup still releases
|
|
3603
|
-
// tracked buffers when GPU work completes. Profile runs keep the
|
|
3604
|
-
// sync path because resolveProfileTimings requires the submitted
|
|
3605
|
-
// work to be done.
|
|
3606
|
-
const chunkSubmitMode = resolvePrefillChunkSubmitMode(
|
|
3607
|
-
this.#state.runtimeConfig,
|
|
3608
|
-
this.#state.modelConfig
|
|
3609
|
-
);
|
|
3610
|
-
const chunkSubmitStart = performance.now();
|
|
3611
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3612
|
-
if (chunkSubmitMode === 'async' && !opts.profile) {
|
|
3613
|
-
currentRecorder.submit({ cleanup: 'deferred' });
|
|
3614
|
-
} else {
|
|
3615
|
-
await currentRecorder.submitAndWait();
|
|
3616
|
-
await recordProfile(currentRecorder);
|
|
3617
|
-
}
|
|
3618
|
-
prefillSubmitWaitMs += performance.now() - chunkSubmitStart;
|
|
3619
|
-
await traceActivationHealth(
|
|
3620
|
-
`PREFILL_LAYER_${l}_HEALTH`,
|
|
3621
|
-
currentHiddenBuffer,
|
|
3622
|
-
activationDtype,
|
|
3623
|
-
numTokens * config.hiddenSize
|
|
3624
|
-
);
|
|
3625
|
-
if (tracePrefillEnabled) {
|
|
3626
|
-
trace.perf('Prefill chunk submitted', {
|
|
3627
|
-
layer: l,
|
|
3628
|
-
elapsedMs: performance.now() - layerLoopTraceStart,
|
|
3629
|
-
prefillRecordMs,
|
|
3630
|
-
prefillSubmitWaitMs,
|
|
3631
|
-
});
|
|
3632
|
-
}
|
|
3633
|
-
currentRecorder = createRecorder('prefill-chunk');
|
|
3634
|
-
}
|
|
3635
|
-
}
|
|
3636
|
-
if (tracePrefillEnabled) {
|
|
3637
|
-
trace.perf('Prefill layer loop recorded', {
|
|
3638
|
-
numTokens,
|
|
3639
|
-
elapsedMs: performance.now() - layerLoopTraceStart,
|
|
3640
|
-
prefillRecordMs,
|
|
3641
|
-
prefillSubmitWaitMs,
|
|
3642
|
-
});
|
|
3643
|
-
}
|
|
3644
|
-
} finally {
|
|
3645
|
-
context.perLayerInputBuffer = null;
|
|
3646
|
-
if (perLayerInputs) {
|
|
3647
|
-
for (const buffer of perLayerInputs) {
|
|
3648
|
-
releasePerLayerInputBuffer(
|
|
3649
|
-
buffer,
|
|
3650
|
-
currentRecorder,
|
|
3651
|
-
context.decodeBuffers,
|
|
3652
|
-
this.#state.pleCache ?? null
|
|
3653
|
-
);
|
|
3654
|
-
}
|
|
3655
|
-
}
|
|
3656
|
-
releaseSharedAttentionState(context.sharedAttentionState, currentRecorder);
|
|
3657
|
-
this.#state.stats.prefillRecordMs = (this.#state.stats.prefillRecordMs ?? 0) + prefillRecordMs;
|
|
3658
|
-
this.#state.stats.prefillSubmitWaitMs = (this.#state.stats.prefillSubmitWaitMs ?? 0) + prefillSubmitWaitMs;
|
|
3659
|
-
}
|
|
3660
|
-
|
|
3661
|
-
{
|
|
3662
|
-
const tsirFixtureCfg = this.#state.operatorDiagnostics?.tsirFixture ?? null;
|
|
3663
|
-
if (tsirFixtureCfg && Array.isArray(tsirFixtureCfg.pendingReads) && tsirFixtureCfg.pendingReads.length > 0) {
|
|
3664
|
-
if (currentRecorder) {
|
|
3665
|
-
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3666
|
-
currentHiddenBuffer,
|
|
3667
|
-
currentRecorder,
|
|
3668
|
-
'prefill_tsir_drain_carry'
|
|
3669
|
-
);
|
|
3670
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3671
|
-
await currentRecorder.submitAndWait();
|
|
3672
|
-
await recordProfile(currentRecorder);
|
|
3673
|
-
currentRecorder = undefined;
|
|
3674
|
-
}
|
|
3675
|
-
await drainPendingTsirReads(tsirFixtureCfg);
|
|
3676
|
-
}
|
|
3677
|
-
}
|
|
3678
|
-
if (this.#state.finitenessBuffer) {
|
|
3679
|
-
if (currentRecorder) {
|
|
3680
|
-
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3681
|
-
currentHiddenBuffer,
|
|
3682
|
-
currentRecorder,
|
|
3683
|
-
'prefill_finiteness_carry'
|
|
3684
|
-
);
|
|
3685
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3686
|
-
await currentRecorder.submitAndWait();
|
|
3687
|
-
await recordProfile(currentRecorder);
|
|
3688
|
-
currentRecorder = undefined;
|
|
3689
|
-
}
|
|
3690
|
-
const isInfiniteData = await readBuffer(this.#state.finitenessBuffer, 16);
|
|
3691
|
-
const u32 = new Uint32Array(isInfiniteData.buffer, isInfiniteData.byteOffset, 4);
|
|
3692
|
-
const finitenessStatus = parseFinitenessStatusWords(u32, 0);
|
|
3693
|
-
if (finitenessStatus.triggered) {
|
|
3694
|
-
if (isGpuBufferInstance(currentHiddenBuffer)) {
|
|
3695
|
-
releaseBuffer(currentHiddenBuffer);
|
|
3696
|
-
}
|
|
3697
|
-
throw new FinitenessError(`F16 bounds exceeded during prefill${finitenessStatus.metadata}`);
|
|
3698
|
-
}
|
|
3699
|
-
}
|
|
3700
|
-
|
|
3701
|
-
if (benchmarkSubmits) {
|
|
3702
|
-
logSubmitStats(`Prefill (${numTokens} tokens, ${config.numLayers} layers)`);
|
|
3703
|
-
setTrackSubmits(false);
|
|
3704
|
-
}
|
|
3705
|
-
|
|
3706
|
-
if (opts.debug) {
|
|
3707
|
-
log.debug('Pipeline', `LAYER_LOOP_DONE, currentHiddenBuffer type=${currentHiddenBuffer?.constructor?.name}`);
|
|
3708
|
-
if (currentHiddenBuffer && allowReadback('pipeline.prefill.final-hidden')) {
|
|
3709
|
-
const lastTokenOffset = (numTokens - 1) * config.hiddenSize * activationBytes;
|
|
3710
|
-
const sampleSize = config.hiddenSize * activationBytes;
|
|
3711
|
-
const data = decodeReadback(
|
|
3712
|
-
await readBufferSlice(currentHiddenBuffer, lastTokenOffset, sampleSize),
|
|
3713
|
-
activationDtype
|
|
3714
|
-
);
|
|
3715
|
-
const nanCount = Array.from(data).filter(x => !Number.isFinite(x)).length;
|
|
3716
|
-
const nonZero = Array.from(data).filter(x => Number.isFinite(x) && x !== 0).slice(0, 5);
|
|
3717
|
-
log.debug('Pipeline', `FINAL_HIDDEN[pos=${numTokens - 1}]: nan=${nanCount}/${data.length}, sample=[${nonZero.map(x => x.toFixed(4)).join(', ')}]`);
|
|
3718
|
-
}
|
|
3719
|
-
}
|
|
3720
|
-
|
|
3721
|
-
if (hasGpuTimePrefill) {
|
|
3722
|
-
this.#state.stats.gpuTimePrefillMs = gpuTimePrefillMs;
|
|
3723
|
-
}
|
|
3724
|
-
|
|
3725
|
-
if (returnHidden) {
|
|
3726
|
-
if (currentRecorder) {
|
|
3727
|
-
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3728
|
-
currentHiddenBuffer,
|
|
3729
|
-
currentRecorder,
|
|
3730
|
-
'prefill_return_hidden_carry'
|
|
3731
|
-
);
|
|
3732
|
-
}
|
|
3733
|
-
return {
|
|
3734
|
-
numTokens,
|
|
3735
|
-
config,
|
|
3736
|
-
startPos,
|
|
3737
|
-
activationDtype,
|
|
3738
|
-
activationBytes,
|
|
3739
|
-
currentRecorder,
|
|
3740
|
-
recordProfile,
|
|
3741
|
-
debugCheckBuffer,
|
|
3742
|
-
currentHiddenBuffer,
|
|
3743
|
-
};
|
|
3744
|
-
}
|
|
3745
|
-
|
|
3746
|
-
|
|
3747
|
-
let lastLogits;
|
|
3748
|
-
let logitsVocabSize = config.vocabSize;
|
|
3749
|
-
let usedRecordedLogits = false;
|
|
3750
|
-
const lmHead = this.#state.weights.get('lm_head');
|
|
3751
|
-
const canRecordLogits = !!currentRecorder
|
|
3752
|
-
&& selectedLogitTokenIds === null
|
|
3753
|
-
&& !!lmHead
|
|
3754
|
-
&& !isCpuWeightBuffer(lmHead)
|
|
3755
|
-
&& !this.#state.disableRecordedLogits
|
|
3756
|
-
&& numTokens === 1;
|
|
3757
|
-
if (currentRecorder && canRecordLogits) {
|
|
3758
|
-
const logitsTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
3759
|
-
const recorded = await recordLogitsGPU(
|
|
3760
|
-
currentRecorder,
|
|
3761
|
-
currentHiddenBuffer,
|
|
3762
|
-
numTokens,
|
|
3763
|
-
getLogitsWeights(this.#state),
|
|
3764
|
-
getLogitsConfig(this.#state),
|
|
3765
|
-
this.#state.operatorDiagnostics
|
|
3766
|
-
);
|
|
3767
|
-
logitsVocabSize = recorded.vocabSize;
|
|
3768
|
-
usedRecordedLogits = true;
|
|
3769
|
-
|
|
3770
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3771
|
-
await currentRecorder.submitAndWait();
|
|
3772
|
-
await recordProfile(currentRecorder);
|
|
3773
|
-
|
|
3774
|
-
const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: recorded.logitsDtype });
|
|
3775
|
-
const lastLogitsSize = logitsVocabSize * logitsBytes;
|
|
3776
|
-
const lastLogitsOffset = (numTokens - 1) * lastLogitsSize;
|
|
3777
|
-
const logitsData = await readBufferSlice(recorded.logitsBuffer, lastLogitsOffset, lastLogitsSize);
|
|
3778
|
-
releaseBuffer(recorded.logitsBuffer);
|
|
3779
|
-
lastLogits = decodeReadback(logitsData, recorded.logitsDtype);
|
|
3780
|
-
if (tracePrefillEnabled) {
|
|
3781
|
-
trace.perf('Prefill recorded logits complete', {
|
|
3782
|
-
numTokens,
|
|
3783
|
-
vocabSize: logitsVocabSize,
|
|
3784
|
-
elapsedMs: performance.now() - logitsTraceStart,
|
|
3785
|
-
});
|
|
3786
|
-
}
|
|
3787
|
-
|
|
3788
|
-
const health = getLogitsHealth(lastLogits);
|
|
3789
|
-
if (health.nanCount > 0 || health.infCount > 0 || health.nonZeroCount === 0) {
|
|
3790
|
-
log.warn(
|
|
3791
|
-
'Logits',
|
|
3792
|
-
`Recorded logits invalid (nan=${health.nanCount} inf=${health.infCount} nonZero=${health.nonZeroCount}, maxAbs=${health.maxAbs.toFixed(3)}); recomputing without recorder.`
|
|
3793
|
-
);
|
|
3794
|
-
this.#state.disableRecordedLogits = true;
|
|
3795
|
-
this.#state.disableFusedDecode = true;
|
|
3796
|
-
const fallbackLogits = await computeLogits(
|
|
3797
|
-
currentHiddenBuffer,
|
|
3798
|
-
numTokens,
|
|
3799
|
-
getLogitsWeights(this.#state),
|
|
3800
|
-
getLogitsConfig(this.#state),
|
|
3801
|
-
this.#state.useGPU,
|
|
3802
|
-
this.#state.debugFlags,
|
|
3803
|
-
undefined,
|
|
3804
|
-
debugCheckBuffer,
|
|
3805
|
-
this.#state.runtimeConfig.shared.debug.probes,
|
|
3806
|
-
{ lastPositionOnly: true },
|
|
3807
|
-
this.#state.operatorDiagnostics
|
|
3808
|
-
);
|
|
3809
|
-
const fallbackHealth = getLogitsHealth(fallbackLogits);
|
|
3810
|
-
if (fallbackHealth.nanCount > 0 || fallbackHealth.infCount > 0 || fallbackHealth.nonZeroCount === 0) {
|
|
3811
|
-
throw new Error(
|
|
3812
|
-
`[Logits] Fallback logits invalid (nan=${fallbackHealth.nanCount} inf=${fallbackHealth.infCount} nonZero=${fallbackHealth.nonZeroCount}, maxAbs=${fallbackHealth.maxAbs.toFixed(3)}). ` +
|
|
3813
|
-
'This indicates upstream kernel output is NaN/Inf (often prefill attention/matmul).'
|
|
3814
|
-
);
|
|
3815
|
-
}
|
|
3816
|
-
logitsVocabSize = config.vocabSize;
|
|
3817
|
-
usedRecordedLogits = false;
|
|
3818
|
-
lastLogits = fallbackLogits.length === logitsVocabSize
|
|
3819
|
-
? fallbackLogits
|
|
3820
|
-
: extractLastPositionLogits(fallbackLogits, numTokens, logitsVocabSize);
|
|
3821
|
-
}
|
|
3822
|
-
|
|
3823
|
-
releaseBuffer(currentHiddenBuffer);
|
|
3824
|
-
} else {
|
|
3825
|
-
const logitsTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
3826
|
-
if (currentRecorder) {
|
|
3827
|
-
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3828
|
-
currentHiddenBuffer,
|
|
3829
|
-
currentRecorder,
|
|
3830
|
-
'prefill_logits_carry'
|
|
3831
|
-
);
|
|
3832
|
-
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3833
|
-
await currentRecorder.submitAndWait();
|
|
3834
|
-
await recordProfile(currentRecorder);
|
|
3835
|
-
}
|
|
3836
|
-
const logits = await computeLogits(
|
|
3837
|
-
currentHiddenBuffer,
|
|
3838
|
-
numTokens,
|
|
3839
|
-
getLogitsWeights(this.#state),
|
|
3840
|
-
getLogitsConfig(this.#state),
|
|
3841
|
-
this.#state.useGPU,
|
|
3842
|
-
this.#state.debugFlags,
|
|
3843
|
-
undefined,
|
|
3844
|
-
debugCheckBuffer,
|
|
3845
|
-
this.#state.runtimeConfig.shared.debug.probes,
|
|
3846
|
-
{ lastPositionOnly: true, selectedTokenIds: selectedLogitTokenIds },
|
|
3847
|
-
this.#state.operatorDiagnostics
|
|
3848
|
-
);
|
|
3849
|
-
|
|
3850
|
-
if (selectedLogitTokenIds !== null) {
|
|
3851
|
-
lastLogits = logits;
|
|
3852
|
-
} else {
|
|
3853
|
-
lastLogits = logits.length === logitsVocabSize
|
|
3854
|
-
? logits
|
|
3855
|
-
: extractLastPositionLogits(logits, numTokens, logitsVocabSize);
|
|
3856
|
-
}
|
|
3857
|
-
releaseBuffer(currentHiddenBuffer);
|
|
3858
|
-
if (tracePrefillEnabled) {
|
|
3859
|
-
trace.perf('Prefill logits complete', {
|
|
3860
|
-
numTokens,
|
|
3861
|
-
vocabSize: logitsVocabSize,
|
|
3862
|
-
elapsedMs: performance.now() - logitsTraceStart,
|
|
3863
|
-
});
|
|
3864
|
-
}
|
|
3865
|
-
}
|
|
3866
|
-
|
|
3867
|
-
this.#state.currentSeqLen = startPos + numTokens;
|
|
3868
|
-
|
|
3869
|
-
if (usedRecordedLogits) {
|
|
3870
|
-
if (logitsVocabSize < config.vocabSize) {
|
|
3871
|
-
const padded = new Float32Array(config.vocabSize);
|
|
3872
|
-
padded.set(lastLogits);
|
|
3873
|
-
padded.fill(-Infinity, logitsVocabSize);
|
|
3874
|
-
lastLogits = padded;
|
|
3875
|
-
}
|
|
3876
|
-
if (config.finalLogitSoftcapping != null) {
|
|
3877
|
-
applySoftcapping(lastLogits, config.finalLogitSoftcapping);
|
|
3878
|
-
}
|
|
3879
|
-
}
|
|
3880
|
-
|
|
3881
|
-
if (opts.debug && selectedLogitTokenIds === null) {
|
|
3882
|
-
logitsSanity(lastLogits, 'Prefill', (tokens) => resolveTokenText(this.#state.tokenizer, tokens));
|
|
3883
|
-
}
|
|
3884
|
-
if (isTraceEnabled('logits')) {
|
|
3885
|
-
trace.logits(selectedLogitTokenIds === null ? 'PREFILL_LOGITS_HEALTH' : 'PREFILL_SELECTED_LOGITS_HEALTH', getLogitsHealth(lastLogits));
|
|
3886
|
-
}
|
|
3887
|
-
|
|
3888
|
-
if (opts.debug) {
|
|
3889
|
-
if (this.#state.kvCache?.hasGPUCache?.()) {
|
|
3890
|
-
log.debug('Pipeline', `KV cache active after prefill: seqLen=${this.#state.kvCache.getKeyCache(0)?.constructor.name ?? '?'}`);
|
|
3891
|
-
} else {
|
|
3892
|
-
log.warn('Pipeline', `KV cache NOT active after prefill! hasGPUCache=${this.#state.kvCache?.hasGPUCache?.()}`);
|
|
3893
|
-
}
|
|
3894
|
-
}
|
|
3895
|
-
|
|
3896
|
-
if (tracePrefillEnabled) {
|
|
3897
|
-
trace.perf('Prefill phase complete', {
|
|
3898
|
-
numTokens,
|
|
3899
|
-
totalMs: performance.now() - prefillTraceStart,
|
|
3900
|
-
});
|
|
3901
|
-
}
|
|
3902
|
-
|
|
3903
|
-
return lastLogits;
|
|
587
|
+
return _prefillImpl.apply(this, arguments);
|
|
3904
588
|
}
|
|
3905
589
|
|
|
3906
|
-
|
|
3907
590
|
async _decodeStep(currentIds, opts) {
|
|
3908
|
-
|
|
3909
|
-
const stepResult = await this._decodeStepToLogits(currentIds, opts);
|
|
3910
|
-
return this._sampleNextTokenFromLogits(stepResult.logits, currentIds, opts);
|
|
3911
|
-
}
|
|
3912
|
-
const debugCheckBuffer = this.#state.debug
|
|
3913
|
-
? (buffer, label, numTokens, expectedDim) =>
|
|
3914
|
-
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
3915
|
-
: undefined;
|
|
3916
|
-
return decodeStep(this.#state, currentIds, opts, this._getDecodeHelpers(debugCheckBuffer));
|
|
591
|
+
return _decodeStepImpl.apply(this, arguments);
|
|
3917
592
|
}
|
|
3918
593
|
|
|
3919
594
|
async decodeStepLogits(currentIds, options = {}) {
|
|
3920
|
-
|
|
3921
|
-
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
3922
|
-
throw new Error('Generation already in progress');
|
|
3923
|
-
}
|
|
3924
|
-
resetActiveExecutionPlan(this.#state);
|
|
3925
|
-
|
|
3926
|
-
validateCallTimeOptions(options);
|
|
3927
|
-
|
|
3928
|
-
const opts = this._resolveStepOptions(options);
|
|
3929
|
-
return this._decodeStepToLogits(currentIds, opts);
|
|
595
|
+
return decodeStepLogitsImpl.apply(this, arguments);
|
|
3930
596
|
}
|
|
3931
597
|
|
|
3932
598
|
async advanceWithToken(tokenId, options = {}) {
|
|
3933
|
-
|
|
3934
|
-
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
3935
|
-
resetActiveExecutionPlan(this.#state);
|
|
3936
|
-
assertIncrementalDecodeSupport(this.#state, 'advanceWithToken');
|
|
3937
|
-
|
|
3938
|
-
validateCallTimeOptions(options);
|
|
3939
|
-
|
|
3940
|
-
const opts = this._resolveStepOptions(options);
|
|
3941
|
-
const debugCheckBuffer = this.#state.debug
|
|
3942
|
-
? (buffer, label, numTokens, expectedDim) =>
|
|
3943
|
-
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
3944
|
-
: undefined;
|
|
3945
|
-
|
|
3946
|
-
this._assertTokenIdInRange(tokenId, 'advanceWithToken');
|
|
3947
|
-
await advanceWithToken(this.#state, tokenId, opts, this._getDecodeHelpers(debugCheckBuffer));
|
|
599
|
+
return advanceWithTokenImpl.apply(this, arguments);
|
|
3948
600
|
}
|
|
3949
601
|
|
|
3950
602
|
async advanceWithTokenAndEmbedding(tokenId, options = {}) {
|
|
3951
|
-
|
|
3952
|
-
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
3953
|
-
resetActiveExecutionPlan(this.#state);
|
|
3954
|
-
assertIncrementalDecodeSupport(this.#state, 'advanceWithTokenAndEmbedding');
|
|
3955
|
-
|
|
3956
|
-
validateCallTimeOptions(options);
|
|
3957
|
-
|
|
3958
|
-
const opts = this._resolveStepOptions(options);
|
|
3959
|
-
const embeddingMode = resolveAdvanceEmbeddingMode(this.#state, options);
|
|
3960
|
-
const debugCheckBuffer = this.#state.debug
|
|
3961
|
-
? (buffer, label, numTokens, expectedDim) =>
|
|
3962
|
-
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
3963
|
-
: undefined;
|
|
3964
|
-
|
|
3965
|
-
this._assertTokenIdInRange(tokenId, 'advanceWithTokenAndEmbedding');
|
|
3966
|
-
return runAdvanceWithTokenAndEmbedding(
|
|
3967
|
-
this.#state,
|
|
3968
|
-
tokenId,
|
|
3969
|
-
opts,
|
|
3970
|
-
this._getDecodeHelpers(debugCheckBuffer),
|
|
3971
|
-
embeddingMode
|
|
3972
|
-
);
|
|
603
|
+
return advanceWithTokenAndEmbeddingImpl.apply(this, arguments);
|
|
3973
604
|
}
|
|
3974
605
|
|
|
3975
606
|
async _generateNTokensGPU(startToken, N, currentIds, opts) {
|
|
3976
|
-
|
|
3977
|
-
? (buffer, label, numTokens, expectedDim) =>
|
|
3978
|
-
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
3979
|
-
: undefined;
|
|
3980
|
-
return generateNTokensGPU(this.#state, startToken, N, currentIds, opts, this._getDecodeHelpers(debugCheckBuffer));
|
|
607
|
+
return _generateNTokensGPUImpl.apply(this, arguments);
|
|
3981
608
|
}
|
|
3982
609
|
}
|