doppler-gpu 0.4.7 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +93 -144
- package/models/catalog.json +1575 -0
- package/package.json +37 -12
- package/src/cli/doppler-cli.js +28 -7
- package/src/client/doppler-registry.json +2 -2
- package/src/client/runtime/index.js +25 -2
- package/src/client/runtime/model-session.d.ts +31 -1
- package/src/client/runtime/model-session.js +31 -0
- package/src/client/runtime/model-source.d.ts +19 -2
- package/src/client/runtime/model-source.js +45 -5
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
- package/src/config/kernel-path-loader.js +1 -0
- package/src/config/kernels/kernel-ref-digests.js +26 -17
- package/src/config/kernels/registry.json +448 -27
- package/src/config/merge.js +11 -0
- package/src/config/param-validator.js +25 -0
- package/src/config/runtime/default.json +3 -1
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
- package/src/config/runtime/profiles/rerank-stability.json +43 -0
- package/src/config/runtime/profiles/rerank-throughput.json +40 -0
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
- package/src/config/schema/doppler.schema.js +40 -5
- package/src/config/schema/execution-v1.schema.d.ts +41 -0
- package/src/config/schema/execution-v1.schema.js +10 -0
- package/src/config/schema/index.d.ts +3 -1
- package/src/config/schema/speculation-self.schema.d.ts +16 -0
- package/src/config/transforms/capability-transform-resolver.js +7 -1
- package/src/config/transforms/execution-graph-transforms.js +2 -2
- package/src/debug/config.d.ts +4 -0
- package/src/experimental/orchestration/index.d.ts +10 -1
- package/src/experimental/training/suite.js +1 -1
- package/src/formats/rdrr/classification.js +3 -0
- package/src/formats/rdrr/integrity.d.ts +0 -2
- package/src/formats/rdrr/integrity.js +0 -2
- package/src/formats/rdrr/types.d.ts +1 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +75 -16
- package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
- package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
- package/src/gpu/kernels/fused_ffn.d.ts +32 -0
- package/src/gpu/kernels/fused_ffn.js +540 -19
- package/src/gpu/kernels/fused_ffn.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
- package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
- package/src/gpu/kernels/index.d.ts +21 -0
- package/src/gpu/kernels/index.js +13 -0
- package/src/gpu/kernels/linear-attention-core.js +413 -15
- package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
- package/src/gpu/kernels/lm-head-argmax.js +153 -57
- package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
- package/src/gpu/kernels/lm-head-select-logits.js +186 -0
- package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
- package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
- package/src/gpu/kernels/matmul-selection.js +5 -1
- package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
- package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
- package/src/gpu/kernels/rmsnorm-pair.js +113 -0
- package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
- package/src/gpu/kernels/rmsnorm-stats.js +250 -0
- package/src/gpu/kernels/silu.wgsl +7 -1
- package/src/gpu/kernels/silu_f16.wgsl +7 -1
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
- package/src/gpu/tensor.d.ts +1 -0
- package/src/index.d.ts +1 -1
- package/src/index.js +1 -1
- package/src/inference/browser-harness-suite-helpers.js +1 -1
- package/src/inference/browser-harness-text-helpers.js +251 -20
- package/src/inference/browser-harness.js +221 -3
- package/src/inference/kv-cache/base.js +42 -1
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
- package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
- package/src/inference/pipelines/text/attention/projections.js +174 -4
- package/src/inference/pipelines/text/attention/record.js +44 -21
- package/src/inference/pipelines/text/attention/run.js +18 -3
- package/src/inference/pipelines/text/config.d.ts +18 -0
- package/src/inference/pipelines/text/config.js +100 -0
- package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
- package/src/inference/pipelines/text/execution-plan.js +37 -0
- package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
- package/src/inference/pipelines/text/execution-v1.js +122 -8
- package/src/inference/pipelines/text/ffn/dense.js +239 -21
- package/src/inference/pipelines/text/ffn/standard.js +121 -38
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
- package/src/inference/pipelines/text/generator-helpers.js +2 -0
- package/src/inference/pipelines/text/generator-runtime.js +1 -0
- package/src/inference/pipelines/text/generator-steps.js +2 -2
- package/src/inference/pipelines/text/generator.d.ts +18 -1
- package/src/inference/pipelines/text/generator.js +297 -19
- package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
- package/src/inference/pipelines/text/init.d.ts +22 -3
- package/src/inference/pipelines/text/init.js +2 -2
- package/src/inference/pipelines/text/layer.js +87 -8
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
- package/src/inference/pipelines/text/linear-attention.js +173 -16
- package/src/inference/pipelines/text/logits/gpu.js +2 -2
- package/src/inference/pipelines/text/logits/index.js +62 -1
- package/src/inference/pipelines/text/model-load.js +6 -0
- package/src/inference/pipelines/text/ops.js +22 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
- package/src/inference/pipelines/text/probes.js +9 -4
- package/src/inference/pipelines/text/state.d.ts +4 -4
- package/src/inference/pipelines/text/state.js +4 -0
- package/src/inference/pipelines/text/types.d.ts +27 -2
- package/src/inference/pipelines/text.d.ts +21 -1
- package/src/inference/pipelines/text.js +40 -14
- package/src/inference/tokenizers/bundled.js +45 -23
- package/src/inference/tokenizers/types.d.ts +7 -4
- package/src/loader/doppler-loader.js +25 -2
- package/src/loader/index.d.ts +3 -1
- package/src/loader/index.js +2 -0
- package/src/loader/loader-types.d.ts +3 -1
- package/src/loader/manifest-config.d.ts +18 -0
- package/src/loader/manifest-config.js +11 -3
- package/src/loader/shard-cache.d.ts +2 -0
- package/src/loader/shard-cache.js +29 -1
- package/src/models/gemma4.js +8 -0
- package/src/rules/inference/capability-transforms.rules.json +145 -20
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/fused-ffn.rules.json +1 -1
- package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/rules/kernels/silu.rules.json +1 -0
- package/src/rules/rule-registry.js +2 -0
- package/src/storage/artifact-storage-context.js +36 -0
- package/src/tooling/node-browser-command-runner.js +42 -7
- package/src/tooling/opfs-cache.d.ts +21 -1
- package/src/tooling/opfs-cache.js +168 -32
- package/src/tooling-exports/storage.d.ts +1 -1
- package/src/tooling-exports/storage.js +1 -1
- package/src/types/gpu.d.ts +25 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +1593 -0
- package/tests/kernels/browser/test-page.js +2629 -0
- package/tests/kernels/harness/benchmark.js +225 -0
- package/tests/kernels/harness/buffer-utils.js +88 -0
- package/tests/kernels/harness/tolerance.js +174 -0
- package/tests/kernels/reference/adam.js +15 -0
- package/tests/kernels/reference/attention.js +158 -0
- package/tests/kernels/reference/backward.js +96 -0
- package/tests/kernels/reference/bias-add.js +20 -0
- package/tests/kernels/reference/conv2d.js +58 -0
- package/tests/kernels/reference/cross-entropy.js +26 -0
- package/tests/kernels/reference/dequant.js +312 -0
- package/tests/kernels/reference/embed.js +15 -0
- package/tests/kernels/reference/gather.js +61 -0
- package/tests/kernels/reference/gelu.js +37 -0
- package/tests/kernels/reference/groupnorm.js +36 -0
- package/tests/kernels/reference/index.js +89 -0
- package/tests/kernels/reference/layernorm.js +22 -0
- package/tests/kernels/reference/matmul.js +57 -0
- package/tests/kernels/reference/modulate.js +24 -0
- package/tests/kernels/reference/moe-gather.js +83 -0
- package/tests/kernels/reference/pixel-shuffle.js +20 -0
- package/tests/kernels/reference/residual.js +55 -0
- package/tests/kernels/reference/rmsnorm.js +52 -0
- package/tests/kernels/reference/rope.js +79 -0
- package/tests/kernels/reference/sample.js +97 -0
- package/tests/kernels/reference/scatter-add.js +58 -0
- package/tests/kernels/reference/silu.js +52 -0
- package/tests/kernels/reference/softmax.js +86 -0
- package/tests/kernels/reference/split-qkv.js +58 -0
- package/tests/kernels/reference/topk.js +88 -0
- package/tests/kernels/reference/upsample2d.js +37 -0
- package/BRANDING.md +0 -14
- package/SECURITY.md +0 -19
- package/assets/doppler-runtime-map.svg +0 -82
- package/src/bootstrap.js +0 -30
- package/src/cli/doppler-cli.d.ts +0 -46
- package/src/cli/doppler-quickstart.d.ts +0 -45
- package/src/cli/doppler-serve.d.ts +0 -210
- package/src/cli/runtime-profiles.d.ts +0 -40
- package/src/client/doppler-api.browser.d.ts +0 -1
- package/src/client/doppler-api.browser.js +0 -25
- package/src/client/doppler-registry.d.ts +0 -54
- package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
- package/src/client/runtime/source-runtime.d.ts +0 -26
- package/src/client/runtime/storage.d.ts +0 -5
- package/src/config/README.md +0 -61
- package/src/config/conversion/README.md +0 -62
- package/src/config/goal-completion-matrix.json +0 -552
- package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
- package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
- package/src/config/kernels/moe/mixtral.paths.json +0 -46
- package/src/config/kernels/registry.d.ts +0 -51
- package/src/config/merge-contract-check.d.ts +0 -16
- package/src/config/merge-contract-check.js +0 -319
- package/src/config/merge-helpers.d.ts +0 -60
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
- package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
- package/src/config/param-categories.d.ts +0 -8
- package/src/config/param-validator.d.ts +0 -33
- package/src/config/platforms/loader.d.ts +0 -38
- package/src/config/quantization-contract-check.d.ts +0 -12
- package/src/config/quantization-contract-check.js +0 -91
- package/src/config/runtime/README.md +0 -28
- package/src/config/runtime-merge.d.ts +0 -5
- package/src/config/schema/claim-evidence-contract.schema.json +0 -93
- package/src/config/schema/command-surface-contract.schema.json +0 -81
- package/src/config/schema/goal-completion-matrix.schema.json +0 -223
- package/src/config/schema/model-artifact-contract.schema.json +0 -119
- package/src/config/schema/platform.schema.js +0 -1
- package/src/config/schema/policy-schema-registry.json +0 -31
- package/src/config/schema/subsystem-support-contract.schema.json +0 -48
- package/src/config/support-tiers/subsystems.json +0 -502
- package/src/config/supported-operations.d.ts +0 -14
- package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
- package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
- package/src/converter/conversion-plan.d.ts +0 -56
- package/src/converter/index.d.ts +0 -98
- package/src/converter/index.js +0 -59
- package/src/converter/parsers/diffusion.d.ts +0 -50
- package/src/converter/parsers/gguf.d.ts +0 -31
- package/src/converter/parsers/index.d.ts +0 -21
- package/src/converter/parsers/index.js +0 -12
- package/src/converter/parsers/transformer.d.ts +0 -31
- package/src/converter/quantization-info.d.ts +0 -38
- package/src/converter/rope-config.d.ts +0 -16
- package/src/converter/tokenizer-utils.d.ts +0 -12
- package/src/debug/reference/README.md +0 -114
- package/src/debug/reference/hf_attn_debug.py +0 -114
- package/src/debug/reference/hf_embed_check.py +0 -89
- package/src/debug/reference/hf_layer_out.py +0 -100
- package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
- package/src/debug/reference/hf_rope_check.py +0 -116
- package/src/debug/reference/hf_weights.py +0 -75
- package/src/errors/doppler-error.d.ts +0 -20
- package/src/experimental/bridge/extension/background.d.ts +0 -14
- package/src/experimental/bridge/extension/background.js +0 -168
- package/src/experimental/bridge/extension/manifest.json +0 -34
- package/src/experimental/bridge/extension-client.d.ts +0 -114
- package/src/experimental/bridge/index.d.ts +0 -69
- package/src/experimental/bridge/node-bridge.js +0 -24
- package/src/experimental/bridge/protocol.d.ts +0 -96
- package/src/experimental/browser/gguf-importer.d.ts +0 -136
- package/src/experimental/browser/gguf-importer.js +0 -532
- package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
- package/src/experimental/browser/quantization.d.ts +0 -74
- package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
- package/src/experimental/browser/shard-io-browser.d.ts +0 -57
- package/src/experimental/browser/shard-io-browser.js +0 -89
- package/src/experimental/browser/tensor-source-download.d.ts +0 -27
- package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
- package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
- package/src/experimental/distribution/p2p-transport.js +0 -75
- package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
- package/src/experimental/hotswap/manifest.d.ts +0 -42
- package/src/experimental/hotswap/model-swap.js +0 -42
- package/src/experimental/hotswap/runtime.d.ts +0 -31
- package/src/experimental/training/README.md +0 -164
- package/src/experimental/training/artifacts.d.ts +0 -160
- package/src/experimental/training/attention-backward.d.ts +0 -30
- package/src/experimental/training/attention-forward.d.ts +0 -22
- package/src/experimental/training/attention-forward.js +0 -82
- package/src/experimental/training/autograd.d.ts +0 -51
- package/src/experimental/training/backward-kernels.js +0 -39
- package/src/experimental/training/checkpoint-watch.d.ts +0 -8
- package/src/experimental/training/checkpoint.d.ts +0 -36
- package/src/experimental/training/clip.d.ts +0 -9
- package/src/experimental/training/dataloader.d.ts +0 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
- package/src/experimental/training/datasets/gepa-frontier.js +0 -220
- package/src/experimental/training/datasets/index.d.ts +0 -24
- package/src/experimental/training/datasets/index.js +0 -13
- package/src/experimental/training/datasets/jsonl.d.ts +0 -11
- package/src/experimental/training/datasets/reploid.d.ts +0 -3
- package/src/experimental/training/datasets/reploid.js +0 -36
- package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
- package/src/experimental/training/datasets/teacher-traces.js +0 -279
- package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
- package/src/experimental/training/datasets/token-batch.d.ts +0 -21
- package/src/experimental/training/datasets/token-batch.js +0 -52
- package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
- package/src/experimental/training/datasets/translation-pairs.js +0 -49
- package/src/experimental/training/distillation/artifacts.d.ts +0 -71
- package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
- package/src/experimental/training/distillation/dataset.d.ts +0 -59
- package/src/experimental/training/distillation/eval.d.ts +0 -34
- package/src/experimental/training/distillation/index.d.ts +0 -29
- package/src/experimental/training/distillation/runtime.d.ts +0 -20
- package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
- package/src/experimental/training/distillation/stage-a.d.ts +0 -45
- package/src/experimental/training/distillation/stage-b.d.ts +0 -24
- package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
- package/src/experimental/training/distillation/suite-data.d.ts +0 -45
- package/src/experimental/training/export.d.ts +0 -41
- package/src/experimental/training/index.d.ts +0 -74
- package/src/experimental/training/index.js +0 -63
- package/src/experimental/training/lora-pipeline.d.ts +0 -183
- package/src/experimental/training/lora.d.ts +0 -19
- package/src/experimental/training/loss-scaling.d.ts +0 -21
- package/src/experimental/training/loss.d.ts +0 -10
- package/src/experimental/training/objectives/base.d.ts +0 -58
- package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
- package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
- package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
- package/src/experimental/training/objectives/index.d.ts +0 -12
- package/src/experimental/training/objectives/index.js +0 -6
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
- package/src/experimental/training/operator-agent-eval.d.ts +0 -65
- package/src/experimental/training/operator-artifacts.d.ts +0 -62
- package/src/experimental/training/operator-command.d.ts +0 -5
- package/src/experimental/training/operator-eval.d.ts +0 -48
- package/src/experimental/training/operator-scoreboard.d.ts +0 -5
- package/src/experimental/training/optimizer.d.ts +0 -22
- package/src/experimental/training/runner.d.ts +0 -249
- package/src/experimental/training/suite.d.ts +0 -299
- package/src/experimental/training/tensor-factory.d.ts +0 -9
- package/src/experimental/training/trainer.d.ts +0 -89
- package/src/experimental/training/ul_dataset.d.ts +0 -47
- package/src/experimental/training/ul_schedule.d.ts +0 -6
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
- package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
- package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
- package/src/experimental/training/workload-packs/registry.json +0 -278
- package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
- package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
- package/src/experimental/training/workloads.d.ts +0 -201
- package/src/formats/gguf/types.d.ts +0 -74
- package/src/formats/index.d.ts +0 -75
- package/src/formats/index.js +0 -19
- package/src/formats/litert/types.d.ts +0 -74
- package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
- package/src/formats/rdrr/distributed/validation.d.ts +0 -23
- package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
- package/src/formats/safetensors/types.d.ts +0 -67
- package/src/formats/tflite/types.d.ts +0 -61
- package/src/formats/tokenizer/index.d.ts +0 -5
- package/src/formats/tokenizer/index.js +0 -3
- package/src/formats/tokenizer/types.d.ts +0 -9
- package/src/gpu/kernel-runtime.d.ts +0 -13
- package/src/gpu/kernel-selection-cache.d.ts +0 -13
- package/src/gpu/kernel-selection-log.d.ts +0 -4
- package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
- package/src/gpu/kernel-tuner/cache.d.ts +0 -55
- package/src/gpu/kernel-tuner/index.d.ts +0 -59
- package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
- package/src/gpu/kernel-tuner/types.d.ts +0 -101
- package/src/gpu/kernel-tuner/types.js +0 -4
- package/src/gpu/kernel-tuner.d.ts +0 -33
- package/src/gpu/kernels/README.md +0 -166
- package/src/gpu/kernels/check-finiteness.d.ts +0 -15
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
- package/src/gpu/kernels/check-stop.d.ts +0 -31
- package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
- package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
- package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
- package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
- package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
- package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
- package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
- package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
- package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
- package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
- package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
- package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
- package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
- package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
- package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
- package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
- package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
- package/src/gpu/kernels/constants.d.ts +0 -129
- package/src/gpu/kernels/dispatch.d.ts +0 -72
- package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
- package/src/gpu/kernels/kernel-base.d.ts +0 -33
- package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
- package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
- package/src/gpu/kernels/matmul-selection.d.ts +0 -87
- package/src/gpu/kernels/rep-penalty.d.ts +0 -21
- package/src/gpu/kernels/rule-matcher.d.ts +0 -30
- package/src/gpu/kernels/rule-registry.d.ts +0 -7
- package/src/gpu/kernels/types.js +0 -4
- package/src/gpu/perf-guards.d.ts +0 -13
- package/src/gpu/profiler.d.ts +0 -93
- package/src/gpu/readback-utils.d.ts +0 -16
- package/src/gpu/readback-utils.js +0 -49
- package/src/gpu/submit-probe.d.ts +0 -12
- package/src/index-browser.d.ts +0 -2
- package/src/index-browser.js +0 -2
- package/src/inference/README.md +0 -593
- package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
- package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
- package/src/inference/browser-harness-model-helpers.d.ts +0 -10
- package/src/inference/browser-harness-report-helpers.d.ts +0 -7
- package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
- package/src/inference/browser-harness-text-helpers.d.ts +0 -152
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
- package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
- package/src/inference/network-evolution.js +0 -79
- package/src/inference/pipelines/context.d.ts +0 -21
- package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
- package/src/inference/pipelines/diffusion/index.d.ts +0 -3
- package/src/inference/pipelines/diffusion/index.js +0 -3
- package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
- package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
- package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
- package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
- package/src/inference/pipelines/diffusion/types.js +0 -1
- package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
- package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
- package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
- package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
- package/src/inference/pipelines/dream/pipeline.js +0 -8
- package/src/inference/pipelines/energy/index.d.ts +0 -1
- package/src/inference/pipelines/energy/index.js +0 -1
- package/src/inference/pipelines/energy/types.js +0 -1
- package/src/inference/pipelines/energy-head/index.d.ts +0 -6
- package/src/inference/pipelines/energy-head/index.js +0 -6
- package/src/inference/pipelines/factory.d.ts +0 -10
- package/src/inference/pipelines/index.d.ts +0 -22
- package/src/inference/pipelines/index.js +0 -19
- package/src/inference/pipelines/registry.d.ts +0 -15
- package/src/inference/pipelines/rng.d.ts +0 -2
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
- package/src/inference/pipelines/text/attention/index.d.ts +0 -24
- package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
- package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
- package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
- package/src/inference/pipelines/text/attention/record.d.ts +0 -36
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
- package/src/inference/pipelines/text/attention/run.d.ts +0 -38
- package/src/inference/pipelines/text/attention/types.d.ts +0 -125
- package/src/inference/pipelines/text/attention.d.ts +0 -23
- package/src/inference/pipelines/text/attention.js +0 -12
- package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
- package/src/inference/pipelines/text/buffer-types.js +0 -4
- package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
- package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
- package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
- package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
- package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
- package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
- package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
- package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
- package/src/inference/pipelines/text/embed.d.ts +0 -69
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
- package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
- package/src/inference/pipelines/text/failure-reduction.js +0 -180
- package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
- package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
- package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
- package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
- package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
- package/src/inference/pipelines/text/ffn.d.ts +0 -31
- package/src/inference/pipelines/text/ffn.js +0 -18
- package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
- package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
- package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
- package/src/inference/pipelines/text/index.d.ts +0 -5
- package/src/inference/pipelines/text/index.js +0 -6
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
- package/src/inference/pipelines/text/layer.d.ts +0 -118
- package/src/inference/pipelines/text/load-errors.d.ts +0 -21
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
- package/src/inference/pipelines/text/logits/types.js +0 -4
- package/src/inference/pipelines/text/logits.d.ts +0 -27
- package/src/inference/pipelines/text/logits.js +0 -16
- package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
- package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
- package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
- package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
- package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
- package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
- package/src/inference/pipelines/text/operator-events.d.ts +0 -125
- package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
- package/src/inference/pipelines/text/ops.d.ts +0 -222
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
- package/src/inference/pipelines/text/probes.d.ts +0 -28
- package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
- package/src/inference/pipelines/text/stage-names.d.ts +0 -88
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
- package/src/inference/pipelines/text/types.js +0 -4
- package/src/inference/tokenizers/base.d.ts +0 -44
- package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
- package/src/inference/tokenizers/bpe.d.ts +0 -27
- package/src/inference/tokenizers/bundled.d.ts +0 -67
- package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
- package/src/inference/tokenizers/types.js +0 -7
- package/src/loader/embedding-limit-preflight.d.ts +0 -31
- package/src/loader/per-layer-input-loader.d.ts +0 -30
- package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
- package/src/loader/tensors/source-transform.d.ts +0 -65
- package/src/memory/address-table.d.ts +0 -96
- package/src/rules/execution-rules-contract-check.d.ts +0 -21
- package/src/rules/layer-pattern-contract-check.d.ts +0 -17
- package/src/rules/rule-registry.d.ts +0 -77
- package/src/storage/artifact-storage-context.d.ts +0 -63
- package/src/storage/backends/idb-store.d.ts +0 -52
- package/src/storage/backends/memory-store.d.ts +0 -36
- package/src/storage/backends/opfs-store.d.ts +0 -41
- package/src/storage/blake3.d.ts +0 -17
- package/src/storage/download-types.d.ts +0 -157
- package/src/storage/downloader.d.ts +0 -103
- package/src/storage/index.d.ts +0 -274
- package/src/storage/index.js +0 -201
- package/src/storage/preflight.d.ts +0 -144
- package/src/storage/preflight.js +0 -360
- package/src/storage/quickstart-downloader.d.ts +0 -157
- package/src/storage/quickstart-downloader.js +0 -289
- package/src/storage/source-artifact-store.d.ts +0 -59
- package/src/sw.js +0 -184
- package/src/tooling/command-api-constants.d.ts +0 -9
- package/src/tooling/command-api-family-normalizers.d.ts +0 -10
- package/src/tooling/command-api-helpers.d.ts +0 -23
- package/src/tooling/command-runner-shared.d.ts +0 -89
- package/src/tooling/conversion-config-materializer.d.ts +0 -14
- package/src/tooling/conversion-config-materializer.js +0 -96
- package/src/tooling/diagnose-runner.d.ts +0 -28
- package/src/tooling/hf-registry-utils.d.ts +0 -55
- package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
- package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
- package/src/tooling/kernel-path-builder/index.d.ts +0 -65
- package/src/tooling/kernel-path-builder/index.js +0 -1361
- package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
- package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
- package/src/tooling/lean-execution-contract-runner.js +0 -158
- package/src/tooling/lean-execution-contract.d.ts +0 -16
- package/src/tooling/lean-execution-contract.js +0 -81
- package/src/tooling/litert-package-runtime.d.ts +0 -46
- package/src/tooling/node-convert-worker-pool.d.ts +0 -16
- package/src/tooling/node-convert-worker.d.ts +0 -1
- package/src/tooling/node-converter.d.ts +0 -1
- package/src/tooling/node-file-fetch.d.ts +0 -1
- package/src/tooling/node-source-runtime.d.ts +0 -24
- package/src/tooling/node-webgpu.d.ts +0 -17
- package/src/tooling/precision-replay-math.js +0 -174
- package/src/tooling/program-bundle-host.d.ts +0 -16
- package/src/tooling/program-bundle-host.js +0 -20
- package/src/tooling/runtime-input-composition.d.ts +0 -37
- package/src/tooling/source-package-profiles.d.ts +0 -50
- package/src/tooling/source-runtime-converter-config.d.ts +0 -19
- package/src/tooling/source-runtime-materializer.d.ts +0 -6
- package/src/tooling/source-runtime-materializer.js +0 -89
- package/src/tooling-exports.browser.d.ts +0 -7
- package/src/types/chrome.d.ts +0 -36
- package/src/types/chrome.js +0 -1
- package/src/types/index.d.ts +0 -1
- package/src/types/index.js +0 -1
- package/src/utils/array-buffer.d.ts +0 -4
- package/src/utils/canonical-hash.d.ts +0 -7
- package/src/utils/clone-json.d.ts +0 -1
- package/src/utils/hf-resolve-url.d.ts +0 -16
- package/src/utils/index.d.ts +0 -10
- package/src/utils/index.js +0 -10
- package/src/utils/load-json.d.ts +0 -5
- package/src/utils/plain-object.d.ts +0 -1
- package/src/utils/sha256.d.ts +0 -5
- package/src/utils/stable-sort-object.d.ts +0 -1
|
@@ -134,6 +134,36 @@ const FINITENESS_RESET_WORDS = new Uint32Array(4);
|
|
|
134
134
|
const tokenizerSuppressionCache = new WeakMap();
|
|
135
135
|
const PREFILL_CHUNK_SUBMIT_MODES = new Set(['sync', 'async']);
|
|
136
136
|
|
|
137
|
+
function mergeRecordOpLabelCounts(target, source) {
|
|
138
|
+
const merged = target && typeof target === 'object' && !Array.isArray(target)
|
|
139
|
+
? target
|
|
140
|
+
: {};
|
|
141
|
+
if (!source || typeof source !== 'object' || Array.isArray(source)) {
|
|
142
|
+
return merged;
|
|
143
|
+
}
|
|
144
|
+
for (const [label, rawCount] of Object.entries(source)) {
|
|
145
|
+
const count = Number(rawCount);
|
|
146
|
+
if (typeof label !== 'string' || label.length === 0 || !Number.isFinite(count) || count <= 0) {
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
merged[label] = (Number.isFinite(merged[label]) ? merged[label] : 0) + count;
|
|
150
|
+
}
|
|
151
|
+
return merged;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function recordPrefillRecorderStats(state, recorder) {
|
|
155
|
+
if (!recorder || typeof recorder.getStats !== 'function') {
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
const recordStats = recorder.getStats();
|
|
159
|
+
state.stats.prefillRecordOps = (state.stats.prefillRecordOps ?? 0) + recordStats.opCount;
|
|
160
|
+
state.stats.prefillRecordPasses = (state.stats.prefillRecordPasses ?? 0) + recordStats.computePassCount;
|
|
161
|
+
state.stats.prefillRecordOpLabels = mergeRecordOpLabelCounts(
|
|
162
|
+
state.stats.prefillRecordOpLabels,
|
|
163
|
+
recordStats.opLabelCounts
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
|
|
137
167
|
export function resolvePrefillChunkSubmitMode(runtimeConfig, modelConfig) {
|
|
138
168
|
const runtimeSubmit = runtimeConfig?.inference?.session?.prefillChunkSubmitMode;
|
|
139
169
|
const manifestSubmit = modelConfig?.sessionSettings?.prefillChunkSubmitMode;
|
|
@@ -219,6 +249,26 @@ function resolveSuppressedSamplingTokenIds(state, samplingConfig) {
|
|
|
219
249
|
return [...suppressed];
|
|
220
250
|
}
|
|
221
251
|
|
|
252
|
+
function normalizeSelectedLogitTokenIds(value, vocabSize, label) {
|
|
253
|
+
if (value == null) {
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
if (!Array.isArray(value) && !ArrayBuffer.isView(value)) {
|
|
257
|
+
throw new Error(`${label} must be an array or typed array.`);
|
|
258
|
+
}
|
|
259
|
+
const tokenIds = Array.from(value, (entry, index) => {
|
|
260
|
+
const tokenId = Number(entry);
|
|
261
|
+
if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSize) {
|
|
262
|
+
throw new Error(`${label}[${index}] must be an integer in [0, ${vocabSize}), got "${String(entry)}".`);
|
|
263
|
+
}
|
|
264
|
+
return tokenId;
|
|
265
|
+
});
|
|
266
|
+
if (tokenIds.length === 0) {
|
|
267
|
+
throw new Error(`${label} must not be empty.`);
|
|
268
|
+
}
|
|
269
|
+
return tokenIds;
|
|
270
|
+
}
|
|
271
|
+
|
|
222
272
|
async function traceActivationHealth(label, buffer, dtype, elementCount) {
|
|
223
273
|
if (!isTraceEnabled('logits') || !isGpuBufferInstance(buffer)) {
|
|
224
274
|
return;
|
|
@@ -535,6 +585,11 @@ export class PipelineGenerator {
|
|
|
535
585
|
this.#state.stats.decodeProfileSteps = [];
|
|
536
586
|
this.#state.stats.ttftMs = 0;
|
|
537
587
|
this.#state.stats.decodeTimeMs = 0;
|
|
588
|
+
this.#state.stats.prefillRecordMs = 0;
|
|
589
|
+
this.#state.stats.prefillRecordOps = 0;
|
|
590
|
+
this.#state.stats.prefillRecordPasses = 0;
|
|
591
|
+
this.#state.stats.prefillRecordOpLabels = {};
|
|
592
|
+
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
538
593
|
this.#state.stats.decodeRecordMs = 0;
|
|
539
594
|
this.#state.stats.decodeRecordOps = 0;
|
|
540
595
|
this.#state.stats.decodeRecordPasses = 0;
|
|
@@ -667,6 +722,7 @@ export class PipelineGenerator {
|
|
|
667
722
|
|
|
668
723
|
try {
|
|
669
724
|
if (currentRecorder) {
|
|
725
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
670
726
|
await currentRecorder.submitAndWait();
|
|
671
727
|
await recordProfile(currentRecorder);
|
|
672
728
|
} else {
|
|
@@ -703,12 +759,15 @@ export class PipelineGenerator {
|
|
|
703
759
|
|
|
704
760
|
async _prefillPromptToLogits(prompt, opts, contextLabel) {
|
|
705
761
|
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
762
|
+
const inputStart = performance.now();
|
|
706
763
|
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, contextLabel, opts.inputIds);
|
|
764
|
+
const inputMs = performance.now() - inputStart;
|
|
707
765
|
if (opts.debug) {
|
|
708
766
|
log.debug('Pipeline', `${contextLabel}: ${inputIds.length} tokens`);
|
|
709
767
|
}
|
|
710
768
|
|
|
711
769
|
let logits;
|
|
770
|
+
const prefillStart = performance.now();
|
|
712
771
|
const runPrefill = () => this._prefillInputIdsToLogits(inputIds, opts);
|
|
713
772
|
try {
|
|
714
773
|
logits = await runPrefill();
|
|
@@ -726,7 +785,21 @@ export class PipelineGenerator {
|
|
|
726
785
|
);
|
|
727
786
|
}
|
|
728
787
|
|
|
729
|
-
return {
|
|
788
|
+
return {
|
|
789
|
+
inputIds,
|
|
790
|
+
logits,
|
|
791
|
+
phase: {
|
|
792
|
+
inputMs,
|
|
793
|
+
prefillMs: performance.now() - prefillStart,
|
|
794
|
+
tokens: inputIds.length,
|
|
795
|
+
prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
|
|
796
|
+
prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
|
|
797
|
+
prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
|
|
798
|
+
prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
|
|
799
|
+
prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
|
|
800
|
+
gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
|
|
801
|
+
},
|
|
802
|
+
};
|
|
730
803
|
}
|
|
731
804
|
|
|
732
805
|
async _decodeStepToLogits(currentIds, opts) {
|
|
@@ -785,6 +858,9 @@ export class PipelineGenerator {
|
|
|
785
858
|
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
786
859
|
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
787
860
|
this.#state.stats.prefillRecordMs = 0;
|
|
861
|
+
this.#state.stats.prefillRecordOps = 0;
|
|
862
|
+
this.#state.stats.prefillRecordPasses = 0;
|
|
863
|
+
this.#state.stats.prefillRecordOpLabels = {};
|
|
788
864
|
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
789
865
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
790
866
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
@@ -1591,6 +1667,7 @@ export class PipelineGenerator {
|
|
|
1591
1667
|
} = prefillResult;
|
|
1592
1668
|
currentHiddenBuffer = prefillResult.currentHiddenBuffer;
|
|
1593
1669
|
if (currentRecorder) {
|
|
1670
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
1594
1671
|
await currentRecorder.submitAndWait();
|
|
1595
1672
|
await recordProfile(currentRecorder);
|
|
1596
1673
|
} else {
|
|
@@ -1678,6 +1755,7 @@ export class PipelineGenerator {
|
|
|
1678
1755
|
} = prefillResult;
|
|
1679
1756
|
currentHiddenBuffer = prefillResult.currentHiddenBuffer;
|
|
1680
1757
|
if (currentRecorder) {
|
|
1758
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
1681
1759
|
await currentRecorder.submitAndWait();
|
|
1682
1760
|
await recordProfile(currentRecorder);
|
|
1683
1761
|
} else {
|
|
@@ -1810,6 +1888,9 @@ export class PipelineGenerator {
|
|
|
1810
1888
|
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
1811
1889
|
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
1812
1890
|
this.#state.stats.prefillRecordMs = 0;
|
|
1891
|
+
this.#state.stats.prefillRecordOps = 0;
|
|
1892
|
+
this.#state.stats.prefillRecordPasses = 0;
|
|
1893
|
+
this.#state.stats.prefillRecordOpLabels = {};
|
|
1813
1894
|
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
1814
1895
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
1815
1896
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
@@ -1991,6 +2072,7 @@ export class PipelineGenerator {
|
|
|
1991
2072
|
|
|
1992
2073
|
// Ensure prefill work completes before returning a usable snapshot.
|
|
1993
2074
|
if (currentRecorder) {
|
|
2075
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
1994
2076
|
await currentRecorder.submitAndWait();
|
|
1995
2077
|
await recordProfile(currentRecorder);
|
|
1996
2078
|
} else {
|
|
@@ -2025,20 +2107,27 @@ export class PipelineGenerator {
|
|
|
2025
2107
|
throw new Error('Generation already in progress');
|
|
2026
2108
|
}
|
|
2027
2109
|
assertIncrementalDecodeSupport(this.#state, 'prefillWithEmbedding');
|
|
2110
|
+
const totalStart = performance.now();
|
|
2028
2111
|
this._resetDecodeRuntimeState();
|
|
2029
2112
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2030
2113
|
this.#state.stats.prefillProfileSteps = [];
|
|
2031
2114
|
const opts = resolvePrefillEmbeddingOptions(this.#state, options);
|
|
2032
2115
|
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
2116
|
+
const inputStart = performance.now();
|
|
2033
2117
|
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
|
|
2118
|
+
const inputMs = performance.now() - inputStart;
|
|
2034
2119
|
if (opts.debug) {
|
|
2035
2120
|
log.debug('Pipeline', `PrefillWithEmbedding: ${inputIds.length} tokens (mode=${opts.embeddingMode})`);
|
|
2036
2121
|
}
|
|
2037
2122
|
|
|
2038
2123
|
try {
|
|
2039
2124
|
let prefillResult;
|
|
2125
|
+
const prefillStart = performance.now();
|
|
2040
2126
|
try {
|
|
2041
|
-
prefillResult = await this._prefillToHidden(inputIds,
|
|
2127
|
+
prefillResult = await this._prefillToHidden(inputIds, {
|
|
2128
|
+
...opts,
|
|
2129
|
+
_embeddingOnly: options.__skipStateSnapshot === true,
|
|
2130
|
+
});
|
|
2042
2131
|
} catch (error) {
|
|
2043
2132
|
if (shouldRetryWithFinitenessFallback(error)) {
|
|
2044
2133
|
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillWithEmbedding. Retrying with F32 precision.`);
|
|
@@ -2046,13 +2135,17 @@ export class PipelineGenerator {
|
|
|
2046
2135
|
opts,
|
|
2047
2136
|
'prefillWithEmbedding',
|
|
2048
2137
|
1,
|
|
2049
|
-
() => this._prefillToHidden(inputIds,
|
|
2138
|
+
() => this._prefillToHidden(inputIds, {
|
|
2139
|
+
...opts,
|
|
2140
|
+
_embeddingOnly: options.__skipStateSnapshot === true,
|
|
2141
|
+
}),
|
|
2050
2142
|
prefillStartSeqLen
|
|
2051
2143
|
);
|
|
2052
2144
|
} else {
|
|
2053
2145
|
throw error;
|
|
2054
2146
|
}
|
|
2055
2147
|
}
|
|
2148
|
+
const prefillMs = performance.now() - prefillStart;
|
|
2056
2149
|
|
|
2057
2150
|
const {
|
|
2058
2151
|
numTokens,
|
|
@@ -2066,7 +2159,10 @@ export class PipelineGenerator {
|
|
|
2066
2159
|
} = prefillResult;
|
|
2067
2160
|
|
|
2068
2161
|
// Ensure prefill work completes before readback.
|
|
2162
|
+
let submitWaitMs = 0;
|
|
2163
|
+
const submitWaitStart = performance.now();
|
|
2069
2164
|
if (currentRecorder) {
|
|
2165
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
2070
2166
|
await currentRecorder.submitAndWait();
|
|
2071
2167
|
await recordProfile(currentRecorder);
|
|
2072
2168
|
} else {
|
|
@@ -2075,21 +2171,34 @@ export class PipelineGenerator {
|
|
|
2075
2171
|
await device.queue.onSubmittedWorkDone();
|
|
2076
2172
|
}
|
|
2077
2173
|
}
|
|
2174
|
+
submitWaitMs = performance.now() - submitWaitStart;
|
|
2078
2175
|
|
|
2079
2176
|
if (!allowReadback('pipeline.prefill.embedding')) {
|
|
2080
2177
|
throw new Error('GPU readback disabled; cannot return embedding');
|
|
2081
2178
|
}
|
|
2082
2179
|
|
|
2083
2180
|
let embedding;
|
|
2181
|
+
let hiddenBytes = 0;
|
|
2182
|
+
let readbackMs = 0;
|
|
2183
|
+
let decodeHiddenMs = 0;
|
|
2184
|
+
let finalNormMs = 0;
|
|
2185
|
+
let extractMs = 0;
|
|
2084
2186
|
try {
|
|
2085
2187
|
const hiddenSize = config.hiddenSize;
|
|
2086
|
-
|
|
2188
|
+
hiddenBytes = numTokens * hiddenSize * activationBytes;
|
|
2189
|
+
const readbackStart = performance.now();
|
|
2087
2190
|
const hiddenData = await readBuffer(currentHiddenBuffer, hiddenBytes);
|
|
2191
|
+
readbackMs = performance.now() - readbackStart;
|
|
2088
2192
|
if (hiddenData.byteLength === 0) {
|
|
2089
2193
|
throw new Error('GPU readback disabled; cannot return embedding');
|
|
2090
2194
|
}
|
|
2195
|
+
const decodeStart = performance.now();
|
|
2091
2196
|
const hiddenStates = decodeReadback(hiddenData, activationDtype);
|
|
2197
|
+
decodeHiddenMs = performance.now() - decodeStart;
|
|
2198
|
+
const finalNormStart = performance.now();
|
|
2092
2199
|
const finalNormWeights = await this._getFinalNormWeights();
|
|
2200
|
+
finalNormMs = performance.now() - finalNormStart;
|
|
2201
|
+
const extractStart = performance.now();
|
|
2093
2202
|
embedding = this._extractEmbeddingFromHidden(
|
|
2094
2203
|
hiddenStates,
|
|
2095
2204
|
numTokens,
|
|
@@ -2098,11 +2207,31 @@ export class PipelineGenerator {
|
|
|
2098
2207
|
finalNormWeights,
|
|
2099
2208
|
config
|
|
2100
2209
|
);
|
|
2210
|
+
extractMs = performance.now() - extractStart;
|
|
2101
2211
|
} finally {
|
|
2102
2212
|
releaseBuffer(currentHiddenBuffer);
|
|
2103
2213
|
}
|
|
2104
2214
|
|
|
2105
2215
|
this.#state.currentSeqLen = startPos + numTokens;
|
|
2216
|
+
const phase = {
|
|
2217
|
+
totalMs: performance.now() - totalStart,
|
|
2218
|
+
inputMs,
|
|
2219
|
+
prefillMs,
|
|
2220
|
+
submitWaitMs,
|
|
2221
|
+
readbackMs,
|
|
2222
|
+
decodeHiddenMs,
|
|
2223
|
+
finalNormMs,
|
|
2224
|
+
extractMs,
|
|
2225
|
+
hiddenBytes,
|
|
2226
|
+
tokens: numTokens,
|
|
2227
|
+
activationDtype,
|
|
2228
|
+
prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
|
|
2229
|
+
prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
|
|
2230
|
+
prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
|
|
2231
|
+
prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
|
|
2232
|
+
prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
|
|
2233
|
+
gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
|
|
2234
|
+
};
|
|
2106
2235
|
|
|
2107
2236
|
// Batch embedding skips expensive KV cache clone and linear attention clone
|
|
2108
2237
|
// since the caller will reset immediately after extracting the embedding.
|
|
@@ -2113,6 +2242,7 @@ export class PipelineGenerator {
|
|
|
2113
2242
|
tokens: inputIds,
|
|
2114
2243
|
embedding,
|
|
2115
2244
|
embeddingMode: opts.embeddingMode,
|
|
2245
|
+
phase,
|
|
2116
2246
|
linearAttention: null,
|
|
2117
2247
|
};
|
|
2118
2248
|
}
|
|
@@ -2128,6 +2258,7 @@ export class PipelineGenerator {
|
|
|
2128
2258
|
tokens: inputIds,
|
|
2129
2259
|
embedding,
|
|
2130
2260
|
embeddingMode: opts.embeddingMode,
|
|
2261
|
+
phase,
|
|
2131
2262
|
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
2132
2263
|
};
|
|
2133
2264
|
} finally {
|
|
@@ -2141,12 +2272,13 @@ export class PipelineGenerator {
|
|
|
2141
2272
|
throw new Error('Generation already in progress');
|
|
2142
2273
|
}
|
|
2143
2274
|
assertIncrementalDecodeSupport(this.#state, 'prefillWithLogits');
|
|
2275
|
+
const totalStart = performance.now();
|
|
2144
2276
|
this._resetDecodeRuntimeState();
|
|
2145
2277
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2146
2278
|
this.#state.stats.prefillProfileSteps = [];
|
|
2147
2279
|
const opts = resolvePrefillOptions(this.#state, options);
|
|
2148
2280
|
try {
|
|
2149
|
-
const { inputIds, logits } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
|
|
2281
|
+
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
|
|
2150
2282
|
|
|
2151
2283
|
const snapshot = this.#state.kvCache?.clone();
|
|
2152
2284
|
if (!snapshot) {
|
|
@@ -2158,6 +2290,10 @@ export class PipelineGenerator {
|
|
|
2158
2290
|
seqLen: this.#state.currentSeqLen,
|
|
2159
2291
|
tokens: inputIds,
|
|
2160
2292
|
logits,
|
|
2293
|
+
phase: {
|
|
2294
|
+
...(prefillPhase ?? {}),
|
|
2295
|
+
totalMs: performance.now() - totalStart,
|
|
2296
|
+
},
|
|
2161
2297
|
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
2162
2298
|
};
|
|
2163
2299
|
} finally {
|
|
@@ -2165,6 +2301,120 @@ export class PipelineGenerator {
|
|
|
2165
2301
|
}
|
|
2166
2302
|
}
|
|
2167
2303
|
|
|
2304
|
+
async prefillWithTokenLogits(prompt, tokenIds, options = {}) {
|
|
2305
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
2306
|
+
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2307
|
+
throw new Error('Generation already in progress');
|
|
2308
|
+
}
|
|
2309
|
+
assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogits');
|
|
2310
|
+
const totalStart = performance.now();
|
|
2311
|
+
const selectedTokenIds = normalizeSelectedLogitTokenIds(
|
|
2312
|
+
tokenIds,
|
|
2313
|
+
this.#state.modelConfig.vocabSize,
|
|
2314
|
+
'prefillWithTokenLogits.tokenIds'
|
|
2315
|
+
);
|
|
2316
|
+
this._resetDecodeRuntimeState();
|
|
2317
|
+
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2318
|
+
this.#state.stats.prefillProfileSteps = [];
|
|
2319
|
+
const opts = resolvePrefillOptions(this.#state, options);
|
|
2320
|
+
opts._selectedLogitTokenIds = selectedTokenIds;
|
|
2321
|
+
try {
|
|
2322
|
+
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogits');
|
|
2323
|
+
const logitsByTokenId = {};
|
|
2324
|
+
for (let index = 0; index < selectedTokenIds.length; index += 1) {
|
|
2325
|
+
logitsByTokenId[selectedTokenIds[index]] = logits[index];
|
|
2326
|
+
}
|
|
2327
|
+
return {
|
|
2328
|
+
seqLen: this.#state.currentSeqLen,
|
|
2329
|
+
tokens: inputIds,
|
|
2330
|
+
tokenIds: selectedTokenIds,
|
|
2331
|
+
logits,
|
|
2332
|
+
logitsByTokenId,
|
|
2333
|
+
phase: {
|
|
2334
|
+
...(prefillPhase ?? {}),
|
|
2335
|
+
totalMs: performance.now() - totalStart,
|
|
2336
|
+
selectedTokenCount: selectedTokenIds.length,
|
|
2337
|
+
},
|
|
2338
|
+
};
|
|
2339
|
+
} finally {
|
|
2340
|
+
this._closeFinitenessFallbackWindow(opts);
|
|
2341
|
+
}
|
|
2342
|
+
}
|
|
2343
|
+
|
|
2344
|
+
async prefillWithTokenLogitsFromKV(prefix, prompt, tokenIds, options = {}) {
|
|
2345
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
2346
|
+
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2347
|
+
throw new Error('Generation already in progress');
|
|
2348
|
+
}
|
|
2349
|
+
assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogitsFromKV');
|
|
2350
|
+
const totalStart = performance.now();
|
|
2351
|
+
if (!prefix || typeof prefix !== 'object' || !prefix.cache) {
|
|
2352
|
+
throw new Error('prefillWithTokenLogitsFromKV.prefix must be a KV cache snapshot.');
|
|
2353
|
+
}
|
|
2354
|
+
if (!Number.isInteger(prefix.seqLen) || prefix.seqLen < 0) {
|
|
2355
|
+
throw new Error('prefillWithTokenLogitsFromKV.prefix.seqLen must be a non-negative integer.');
|
|
2356
|
+
}
|
|
2357
|
+
if (!Array.isArray(prefix.tokens)) {
|
|
2358
|
+
throw new Error('prefillWithTokenLogitsFromKV.prefix.tokens must be an array.');
|
|
2359
|
+
}
|
|
2360
|
+
const selectedTokenIds = normalizeSelectedLogitTokenIds(
|
|
2361
|
+
tokenIds,
|
|
2362
|
+
this.#state.modelConfig.vocabSize,
|
|
2363
|
+
'prefillWithTokenLogitsFromKV.tokenIds'
|
|
2364
|
+
);
|
|
2365
|
+
|
|
2366
|
+
this.#state.kvCache = prefix.cache.clone();
|
|
2367
|
+
if (this.#state.useGPU && this.#state.kvCache) {
|
|
2368
|
+
const device = getDevice();
|
|
2369
|
+
if (device) {
|
|
2370
|
+
this.#state.kvCache.setGPUContext({ device });
|
|
2371
|
+
}
|
|
2372
|
+
}
|
|
2373
|
+
if (
|
|
2374
|
+
hasLinearAttentionLayers(this.#state.modelConfig.layerTypes)
|
|
2375
|
+
&& prefix.linearAttention == null
|
|
2376
|
+
) {
|
|
2377
|
+
throw new Error(
|
|
2378
|
+
'Prefix snapshot is missing linear_attention recurrent state. ' +
|
|
2379
|
+
'Regenerate the prefix snapshot using the current runtime.'
|
|
2380
|
+
);
|
|
2381
|
+
}
|
|
2382
|
+
this.#state.linearAttentionRuntime = restoreLinearAttentionRuntime(
|
|
2383
|
+
this.#state.linearAttentionRuntime,
|
|
2384
|
+
prefix.linearAttention ?? null
|
|
2385
|
+
);
|
|
2386
|
+
this.#state.currentSeqLen = prefix.seqLen;
|
|
2387
|
+
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2388
|
+
this.#state.stats.prefillProfileSteps = [];
|
|
2389
|
+
this.#state.decodeRing?.reset();
|
|
2390
|
+
|
|
2391
|
+
const opts = resolvePrefillOptions(this.#state, options);
|
|
2392
|
+
opts._selectedLogitTokenIds = selectedTokenIds;
|
|
2393
|
+
try {
|
|
2394
|
+
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogitsFromKV');
|
|
2395
|
+
const logitsByTokenId = {};
|
|
2396
|
+
for (let index = 0; index < selectedTokenIds.length; index += 1) {
|
|
2397
|
+
logitsByTokenId[selectedTokenIds[index]] = logits[index];
|
|
2398
|
+
}
|
|
2399
|
+
return {
|
|
2400
|
+
seqLen: this.#state.currentSeqLen,
|
|
2401
|
+
prefixTokens: prefix.tokens,
|
|
2402
|
+
tokens: inputIds,
|
|
2403
|
+
tokenIds: selectedTokenIds,
|
|
2404
|
+
logits,
|
|
2405
|
+
logitsByTokenId,
|
|
2406
|
+
phase: {
|
|
2407
|
+
...(prefillPhase ?? {}),
|
|
2408
|
+
totalMs: performance.now() - totalStart,
|
|
2409
|
+
selectedTokenCount: selectedTokenIds.length,
|
|
2410
|
+
prefixTokens: prefix.tokens.length,
|
|
2411
|
+
},
|
|
2412
|
+
};
|
|
2413
|
+
} finally {
|
|
2414
|
+
this._closeFinitenessFallbackWindow(opts);
|
|
2415
|
+
}
|
|
2416
|
+
}
|
|
2417
|
+
|
|
2168
2418
|
|
|
2169
2419
|
async *generateWithPrefixKV(prefix, prompt, options = {}) {
|
|
2170
2420
|
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
@@ -2212,6 +2462,11 @@ export class PipelineGenerator {
|
|
|
2212
2462
|
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
2213
2463
|
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
2214
2464
|
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
2465
|
+
this.#state.stats.prefillRecordMs = 0;
|
|
2466
|
+
this.#state.stats.prefillRecordOps = 0;
|
|
2467
|
+
this.#state.stats.prefillRecordPasses = 0;
|
|
2468
|
+
this.#state.stats.prefillRecordOpLabels = {};
|
|
2469
|
+
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
2215
2470
|
this.#state.stats.ttftMs = 0;
|
|
2216
2471
|
const startTime = performance.now();
|
|
2217
2472
|
|
|
@@ -2412,6 +2667,7 @@ export class PipelineGenerator {
|
|
|
2412
2667
|
maxDecodeTokens: opts.maxTokens,
|
|
2413
2668
|
numLayers: this.#state.modelConfig.numLayers,
|
|
2414
2669
|
hiddenSize: this.#state.modelConfig.hiddenSize,
|
|
2670
|
+
configuredMaxBatchDecodeTokens: executionPlan.maxBatchDecodeTokens,
|
|
2415
2671
|
});
|
|
2416
2672
|
const requestedBatchTokens = executionPlan.batchSize * intervalBatches;
|
|
2417
2673
|
const boundedBatchTokens = maxBatchDecodeTokens == null
|
|
@@ -2725,6 +2981,11 @@ export class PipelineGenerator {
|
|
|
2725
2981
|
async _prefill(inputIds, opts) {
|
|
2726
2982
|
const numTokens = inputIds.length;
|
|
2727
2983
|
const config = this.#state.modelConfig;
|
|
2984
|
+
const selectedLogitTokenIds = normalizeSelectedLogitTokenIds(
|
|
2985
|
+
opts?._selectedLogitTokenIds,
|
|
2986
|
+
config.vocabSize,
|
|
2987
|
+
'_prefill.selectedLogitTokenIds'
|
|
2988
|
+
);
|
|
2728
2989
|
const startPos = this.#state.currentSeqLen;
|
|
2729
2990
|
const tracePrefillEnabled = isTraceEnabled('perf');
|
|
2730
2991
|
const prefillTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
@@ -2787,7 +3048,7 @@ export class PipelineGenerator {
|
|
|
2787
3048
|
if (!device || disableCommandBatching) return undefined;
|
|
2788
3049
|
return opts.profile
|
|
2789
3050
|
? createProfilingRecorder(label, device)
|
|
2790
|
-
: createCommandRecorder(label, { recordLabels: opts.debug === true }, device);
|
|
3051
|
+
: createCommandRecorder(label, { recordLabels: opts.debug === true || opts.benchmark === true }, device);
|
|
2791
3052
|
};
|
|
2792
3053
|
const recorder = createRecorder('prefill');
|
|
2793
3054
|
const debugCheckBuffer = this.#state.debug
|
|
@@ -2804,6 +3065,9 @@ export class PipelineGenerator {
|
|
|
2804
3065
|
);
|
|
2805
3066
|
context.currentTokenIds = inputIds;
|
|
2806
3067
|
context.diffusionGemmaDecoder = opts?._diffusionGemmaDecoder === true;
|
|
3068
|
+
context.skipKVCacheWrites = returnHidden
|
|
3069
|
+
&& opts?._embeddingOnly === true
|
|
3070
|
+
&& this.#state.runtimeConfig?.inference?.session?.skipEmbeddingKVCacheWrites === true;
|
|
2807
3071
|
context.multimodalBidirectionalSpan = multimodalBidirectionalSpan == null
|
|
2808
3072
|
? null
|
|
2809
3073
|
: {
|
|
@@ -2938,6 +3202,7 @@ export class PipelineGenerator {
|
|
|
2938
3202
|
hiddenStates.shape,
|
|
2939
3203
|
hiddenStates.label
|
|
2940
3204
|
);
|
|
3205
|
+
recordPrefillRecorderStats(this.#state, recorder);
|
|
2941
3206
|
await recorder.submitAndWait();
|
|
2942
3207
|
await recordProfile(recorder);
|
|
2943
3208
|
}
|
|
@@ -2972,6 +3237,7 @@ export class PipelineGenerator {
|
|
|
2972
3237
|
// buffers, preventing unbounded memory growth during large prefills. Critical for
|
|
2973
3238
|
// replay_prefill models where each decode step re-runs a prefill-style layer pass.
|
|
2974
3239
|
const prefillRecorderChunkLayers = resolvePrefillRecorderChunkLayers({
|
|
3240
|
+
configuredPrefillChunkLayers: this.#state.runtimeConfig?.inference?.session?.prefillChunkLayers,
|
|
2975
3241
|
hasGpuSplitPerLayerInputs: context.perLayerInputsSession?.materialization === 'gpu_split_tables',
|
|
2976
3242
|
numTokens,
|
|
2977
3243
|
});
|
|
@@ -3028,6 +3294,7 @@ export class PipelineGenerator {
|
|
|
3028
3294
|
currentRecorder,
|
|
3029
3295
|
'prefill_checkpoint_carry'
|
|
3030
3296
|
);
|
|
3297
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3031
3298
|
await currentRecorder.submitAndWait();
|
|
3032
3299
|
await recordProfile(currentRecorder);
|
|
3033
3300
|
currentRecorder = undefined;
|
|
@@ -3040,6 +3307,7 @@ export class PipelineGenerator {
|
|
|
3040
3307
|
'prefill_trace_layer_health_carry'
|
|
3041
3308
|
);
|
|
3042
3309
|
const traceSubmitStart = performance.now();
|
|
3310
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3043
3311
|
await currentRecorder.submitAndWait();
|
|
3044
3312
|
await recordProfile(currentRecorder);
|
|
3045
3313
|
prefillSubmitWaitMs += performance.now() - traceSubmitStart;
|
|
@@ -3113,6 +3381,7 @@ export class PipelineGenerator {
|
|
|
3113
3381
|
this.#state.modelConfig
|
|
3114
3382
|
);
|
|
3115
3383
|
const chunkSubmitStart = performance.now();
|
|
3384
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3116
3385
|
if (chunkSubmitMode === 'async' && !opts.profile) {
|
|
3117
3386
|
currentRecorder.submit({ cleanup: 'deferred' });
|
|
3118
3387
|
} else {
|
|
@@ -3171,6 +3440,7 @@ export class PipelineGenerator {
|
|
|
3171
3440
|
currentRecorder,
|
|
3172
3441
|
'prefill_tsir_drain_carry'
|
|
3173
3442
|
);
|
|
3443
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3174
3444
|
await currentRecorder.submitAndWait();
|
|
3175
3445
|
await recordProfile(currentRecorder);
|
|
3176
3446
|
currentRecorder = undefined;
|
|
@@ -3181,13 +3451,14 @@ export class PipelineGenerator {
|
|
|
3181
3451
|
if (this.#state.finitenessBuffer) {
|
|
3182
3452
|
if (currentRecorder) {
|
|
3183
3453
|
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3184
|
-
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3454
|
+
currentHiddenBuffer,
|
|
3455
|
+
currentRecorder,
|
|
3456
|
+
'prefill_finiteness_carry'
|
|
3457
|
+
);
|
|
3458
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3459
|
+
await currentRecorder.submitAndWait();
|
|
3460
|
+
await recordProfile(currentRecorder);
|
|
3461
|
+
currentRecorder = undefined;
|
|
3191
3462
|
}
|
|
3192
3463
|
const isInfiniteData = await readBuffer(this.#state.finitenessBuffer, 16);
|
|
3193
3464
|
const u32 = new Uint32Array(isInfiniteData.buffer, isInfiniteData.byteOffset, 4);
|
|
@@ -3251,6 +3522,7 @@ export class PipelineGenerator {
|
|
|
3251
3522
|
let usedRecordedLogits = false;
|
|
3252
3523
|
const lmHead = this.#state.weights.get('lm_head');
|
|
3253
3524
|
const canRecordLogits = !!currentRecorder
|
|
3525
|
+
&& selectedLogitTokenIds === null
|
|
3254
3526
|
&& !!lmHead
|
|
3255
3527
|
&& !isCpuWeightBuffer(lmHead)
|
|
3256
3528
|
&& !this.#state.disableRecordedLogits
|
|
@@ -3268,6 +3540,7 @@ export class PipelineGenerator {
|
|
|
3268
3540
|
logitsVocabSize = recorded.vocabSize;
|
|
3269
3541
|
usedRecordedLogits = true;
|
|
3270
3542
|
|
|
3543
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3271
3544
|
await currentRecorder.submitAndWait();
|
|
3272
3545
|
await recordProfile(currentRecorder);
|
|
3273
3546
|
|
|
@@ -3329,6 +3602,7 @@ export class PipelineGenerator {
|
|
|
3329
3602
|
currentRecorder,
|
|
3330
3603
|
'prefill_logits_carry'
|
|
3331
3604
|
);
|
|
3605
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3332
3606
|
await currentRecorder.submitAndWait();
|
|
3333
3607
|
await recordProfile(currentRecorder);
|
|
3334
3608
|
}
|
|
@@ -3342,13 +3616,17 @@ export class PipelineGenerator {
|
|
|
3342
3616
|
undefined,
|
|
3343
3617
|
debugCheckBuffer,
|
|
3344
3618
|
this.#state.runtimeConfig.shared.debug.probes,
|
|
3345
|
-
{ lastPositionOnly: true },
|
|
3619
|
+
{ lastPositionOnly: true, selectedTokenIds: selectedLogitTokenIds },
|
|
3346
3620
|
this.#state.operatorDiagnostics
|
|
3347
3621
|
);
|
|
3348
3622
|
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3623
|
+
if (selectedLogitTokenIds !== null) {
|
|
3624
|
+
lastLogits = logits;
|
|
3625
|
+
} else {
|
|
3626
|
+
lastLogits = logits.length === logitsVocabSize
|
|
3627
|
+
? logits
|
|
3628
|
+
: extractLastPositionLogits(logits, numTokens, logitsVocabSize);
|
|
3629
|
+
}
|
|
3352
3630
|
releaseBuffer(currentHiddenBuffer);
|
|
3353
3631
|
if (tracePrefillEnabled) {
|
|
3354
3632
|
trace.perf('Prefill logits complete', {
|
|
@@ -3373,11 +3651,11 @@ export class PipelineGenerator {
|
|
|
3373
3651
|
}
|
|
3374
3652
|
}
|
|
3375
3653
|
|
|
3376
|
-
if (opts.debug) {
|
|
3654
|
+
if (opts.debug && selectedLogitTokenIds === null) {
|
|
3377
3655
|
logitsSanity(lastLogits, 'Prefill', (tokens) => resolveTokenText(this.#state.tokenizer, tokens));
|
|
3378
3656
|
}
|
|
3379
3657
|
if (isTraceEnabled('logits')) {
|
|
3380
|
-
trace.logits('PREFILL_LOGITS_HEALTH', getLogitsHealth(lastLogits));
|
|
3658
|
+
trace.logits(selectedLogitTokenIds === null ? 'PREFILL_LOGITS_HEALTH' : 'PREFILL_SELECTED_LOGITS_HEALTH', getLogitsHealth(lastLogits));
|
|
3381
3659
|
}
|
|
3382
3660
|
|
|
3383
3661
|
if (opts.debug) {
|