doppler-gpu 0.4.7 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -0
- package/README.md +93 -144
- package/models/catalog.json +1589 -0
- package/package.json +44 -12
- package/src/cli/doppler-cli.js +28 -7
- package/src/client/doppler-registry.json +2 -2
- package/src/client/runtime/index.js +25 -2
- package/src/client/runtime/model-session.d.ts +31 -1
- package/src/client/runtime/model-session.js +31 -0
- package/src/client/runtime/model-source.d.ts +19 -2
- package/src/client/runtime/model-source.js +45 -5
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
- package/src/config/kernel-path-loader.js +1 -0
- package/src/config/kernels/backward-registry.json +13 -2
- package/src/config/kernels/kernel-ref-digests.js +34 -21
- package/src/config/kernels/registry.json +1402 -2623
- package/src/config/merge.js +11 -0
- package/src/config/param-validator.js +25 -0
- package/src/config/runtime/default.json +3 -1
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
- package/src/config/runtime/profiles/rerank-stability.json +43 -0
- package/src/config/runtime/profiles/rerank-throughput.json +40 -0
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
- package/src/config/schema/doppler.schema.js +40 -5
- package/src/config/schema/execution-v1.schema.d.ts +41 -0
- package/src/config/schema/execution-v1.schema.js +10 -0
- package/src/config/schema/index.d.ts +3 -1
- package/src/config/schema/speculation-self.schema.d.ts +16 -0
- package/src/config/schema/training-metrics.schema.js +4 -0
- package/src/config/transforms/capability-transform-resolver.js +7 -1
- package/src/config/transforms/execution-graph-transforms.js +2 -2
- package/src/debug/config.d.ts +4 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
- package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
- package/src/experimental/orchestration/index.d.ts +10 -1
- package/src/experimental/training/attention-backward.js +123 -55
- package/src/experimental/training/autograd.js +101 -5
- package/src/experimental/training/clip.js +6 -1
- package/src/experimental/training/datasets/text-pairs.js +80 -8
- package/src/experimental/training/distillation/student-fixture.js +470 -114
- package/src/experimental/training/lora-pipeline.js +202 -27
- package/src/experimental/training/lora.js +10 -4
- package/src/experimental/training/optimizer.js +9 -1
- package/src/experimental/training/runner.js +18 -4
- package/src/experimental/training/suite.js +1 -1
- package/src/experimental/training/workloads.js +11 -0
- package/src/formats/rdrr/classification.js +3 -0
- package/src/formats/rdrr/integrity.d.ts +0 -2
- package/src/formats/rdrr/integrity.js +0 -2
- package/src/formats/rdrr/types.d.ts +1 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +75 -16
- package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
- package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
- package/src/gpu/kernels/backward/adam.js +6 -4
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
- package/src/gpu/kernels/backward/index.d.ts +3 -0
- package/src/gpu/kernels/backward/index.js +3 -0
- package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
- package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
- package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
- package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
- package/src/gpu/kernels/backward/utils.d.ts +12 -2
- package/src/gpu/kernels/backward/utils.js +9 -2
- package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
- package/src/gpu/kernels/fused_ffn.d.ts +32 -0
- package/src/gpu/kernels/fused_ffn.js +540 -19
- package/src/gpu/kernels/fused_ffn.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
- package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
- package/src/gpu/kernels/index.d.ts +26 -0
- package/src/gpu/kernels/index.js +18 -0
- package/src/gpu/kernels/linear-attention-core.js +413 -15
- package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
- package/src/gpu/kernels/lm-head-argmax.js +153 -57
- package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
- package/src/gpu/kernels/lm-head-select-logits.js +186 -0
- package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
- package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
- package/src/gpu/kernels/matmul-selection.js +10 -1
- package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
- package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
- package/src/gpu/kernels/rmsnorm-pair.js +113 -0
- package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
- package/src/gpu/kernels/rmsnorm-stats.js +250 -0
- package/src/gpu/kernels/silu.d.ts +1 -0
- package/src/gpu/kernels/silu.js +19 -4
- package/src/gpu/kernels/silu.wgsl +7 -1
- package/src/gpu/kernels/silu_f16.wgsl +7 -1
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
- package/src/gpu/tensor.d.ts +1 -0
- package/src/index.d.ts +1 -1
- package/src/index.js +1 -1
- package/src/inference/browser-harness-suite-helpers.js +1 -1
- package/src/inference/browser-harness-text-helpers.js +251 -20
- package/src/inference/browser-harness.js +221 -3
- package/src/inference/kv-cache/base.js +42 -1
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
- package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
- package/src/inference/pipelines/text/attention/projections.js +174 -4
- package/src/inference/pipelines/text/attention/record.js +44 -21
- package/src/inference/pipelines/text/attention/run.js +18 -3
- package/src/inference/pipelines/text/config.d.ts +18 -0
- package/src/inference/pipelines/text/config.js +100 -0
- package/src/inference/pipelines/text/embed.js +1 -1
- package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
- package/src/inference/pipelines/text/execution-plan.js +37 -0
- package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
- package/src/inference/pipelines/text/execution-v1.js +130 -10
- package/src/inference/pipelines/text/ffn/dense.js +241 -21
- package/src/inference/pipelines/text/ffn/standard.js +121 -38
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
- package/src/inference/pipelines/text/generator-helpers.js +2 -0
- package/src/inference/pipelines/text/generator-runtime.js +1 -0
- package/src/inference/pipelines/text/generator-steps.js +2 -2
- package/src/inference/pipelines/text/generator.d.ts +18 -1
- package/src/inference/pipelines/text/generator.js +326 -19
- package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
- package/src/inference/pipelines/text/init.d.ts +22 -3
- package/src/inference/pipelines/text/init.js +2 -2
- package/src/inference/pipelines/text/layer.js +87 -8
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
- package/src/inference/pipelines/text/linear-attention.js +173 -16
- package/src/inference/pipelines/text/logits/gpu.js +2 -2
- package/src/inference/pipelines/text/logits/index.js +62 -1
- package/src/inference/pipelines/text/lora-apply.js +4 -4
- package/src/inference/pipelines/text/model-load.js +6 -0
- package/src/inference/pipelines/text/moe-cpu.js +1 -0
- package/src/inference/pipelines/text/moe-gpu.js +1 -0
- package/src/inference/pipelines/text/ops.js +22 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
- package/src/inference/pipelines/text/probes.js +9 -4
- package/src/inference/pipelines/text/state.d.ts +4 -4
- package/src/inference/pipelines/text/state.js +4 -0
- package/src/inference/pipelines/text/types.d.ts +27 -2
- package/src/inference/pipelines/text.d.ts +21 -1
- package/src/inference/pipelines/text.js +40 -14
- package/src/inference/tokenizers/bundled.js +150 -55
- package/src/inference/tokenizers/types.d.ts +7 -4
- package/src/loader/doppler-loader.js +25 -2
- package/src/loader/index.d.ts +3 -1
- package/src/loader/index.js +2 -0
- package/src/loader/loader-types.d.ts +3 -1
- package/src/loader/manifest-config.d.ts +18 -0
- package/src/loader/manifest-config.js +11 -3
- package/src/loader/shard-cache.d.ts +2 -0
- package/src/loader/shard-cache.js +29 -1
- package/src/models/gemma4.js +8 -0
- package/src/rules/inference/capability-transforms.rules.json +145 -20
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/fused-ffn.rules.json +1 -1
- package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
- package/src/rules/kernels/matmul.rules.json +2 -0
- package/src/rules/kernels/silu.rules.json +1 -0
- package/src/rules/rule-registry.js +2 -0
- package/src/storage/artifact-storage-context.js +36 -0
- package/src/tooling/node-browser-command-runner.js +42 -7
- package/src/tooling/node-converter.js +20 -5
- package/src/tooling/opfs-cache.d.ts +21 -1
- package/src/tooling/opfs-cache.js +168 -32
- package/src/tooling-experimental-exports.shared.d.ts +25 -0
- package/src/tooling-experimental-exports.shared.js +18 -0
- package/src/tooling-exports/storage.d.ts +1 -1
- package/src/tooling-exports/storage.js +1 -1
- package/src/types/gpu.d.ts +25 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +1593 -0
- package/tests/kernels/browser/test-page.js +2629 -0
- package/tests/kernels/harness/benchmark.js +225 -0
- package/tests/kernels/harness/buffer-utils.js +88 -0
- package/tests/kernels/harness/tolerance.js +174 -0
- package/tests/kernels/reference/adam.js +15 -0
- package/tests/kernels/reference/attention.js +158 -0
- package/tests/kernels/reference/backward.js +96 -0
- package/tests/kernels/reference/bias-add.js +20 -0
- package/tests/kernels/reference/conv2d.js +58 -0
- package/tests/kernels/reference/cross-entropy.js +26 -0
- package/tests/kernels/reference/dequant.js +312 -0
- package/tests/kernels/reference/embed.js +15 -0
- package/tests/kernels/reference/gather.js +61 -0
- package/tests/kernels/reference/gelu.js +37 -0
- package/tests/kernels/reference/groupnorm.js +36 -0
- package/tests/kernels/reference/index.js +89 -0
- package/tests/kernels/reference/layernorm.js +22 -0
- package/tests/kernels/reference/matmul.js +57 -0
- package/tests/kernels/reference/modulate.js +24 -0
- package/tests/kernels/reference/moe-gather.js +83 -0
- package/tests/kernels/reference/pixel-shuffle.js +20 -0
- package/tests/kernels/reference/residual.js +55 -0
- package/tests/kernels/reference/rmsnorm.js +52 -0
- package/tests/kernels/reference/rope.js +79 -0
- package/tests/kernels/reference/sample.js +97 -0
- package/tests/kernels/reference/scatter-add.js +58 -0
- package/tests/kernels/reference/silu.js +52 -0
- package/tests/kernels/reference/softmax.js +86 -0
- package/tests/kernels/reference/split-qkv.js +58 -0
- package/tests/kernels/reference/topk.js +88 -0
- package/tests/kernels/reference/upsample2d.js +37 -0
- package/BRANDING.md +0 -14
- package/SECURITY.md +0 -19
- package/assets/doppler-runtime-map.svg +0 -82
- package/src/bootstrap.js +0 -30
- package/src/cli/doppler-cli.d.ts +0 -46
- package/src/cli/doppler-quickstart.d.ts +0 -45
- package/src/cli/doppler-serve.d.ts +0 -210
- package/src/cli/runtime-profiles.d.ts +0 -40
- package/src/client/doppler-api.browser.d.ts +0 -1
- package/src/client/doppler-api.browser.js +0 -25
- package/src/client/doppler-registry.d.ts +0 -54
- package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
- package/src/client/runtime/source-runtime.d.ts +0 -26
- package/src/client/runtime/storage.d.ts +0 -5
- package/src/config/README.md +0 -61
- package/src/config/conversion/README.md +0 -62
- package/src/config/goal-completion-matrix.json +0 -552
- package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
- package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
- package/src/config/kernels/moe/mixtral.paths.json +0 -46
- package/src/config/kernels/registry.d.ts +0 -51
- package/src/config/merge-contract-check.d.ts +0 -16
- package/src/config/merge-contract-check.js +0 -319
- package/src/config/merge-helpers.d.ts +0 -60
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
- package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
- package/src/config/param-categories.d.ts +0 -8
- package/src/config/param-validator.d.ts +0 -33
- package/src/config/platforms/loader.d.ts +0 -38
- package/src/config/quantization-contract-check.d.ts +0 -12
- package/src/config/quantization-contract-check.js +0 -91
- package/src/config/runtime/README.md +0 -28
- package/src/config/runtime-merge.d.ts +0 -5
- package/src/config/schema/claim-evidence-contract.schema.json +0 -93
- package/src/config/schema/command-surface-contract.schema.json +0 -81
- package/src/config/schema/goal-completion-matrix.schema.json +0 -223
- package/src/config/schema/model-artifact-contract.schema.json +0 -119
- package/src/config/schema/platform.schema.js +0 -1
- package/src/config/schema/policy-schema-registry.json +0 -31
- package/src/config/schema/subsystem-support-contract.schema.json +0 -48
- package/src/config/support-tiers/subsystems.json +0 -502
- package/src/config/supported-operations.d.ts +0 -14
- package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
- package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
- package/src/converter/conversion-plan.d.ts +0 -56
- package/src/converter/index.d.ts +0 -98
- package/src/converter/index.js +0 -59
- package/src/converter/parsers/diffusion.d.ts +0 -50
- package/src/converter/parsers/gguf.d.ts +0 -31
- package/src/converter/parsers/index.d.ts +0 -21
- package/src/converter/parsers/index.js +0 -12
- package/src/converter/parsers/transformer.d.ts +0 -31
- package/src/converter/quantization-info.d.ts +0 -38
- package/src/converter/rope-config.d.ts +0 -16
- package/src/converter/tokenizer-utils.d.ts +0 -12
- package/src/debug/reference/README.md +0 -114
- package/src/debug/reference/hf_attn_debug.py +0 -114
- package/src/debug/reference/hf_embed_check.py +0 -89
- package/src/debug/reference/hf_layer_out.py +0 -100
- package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
- package/src/debug/reference/hf_rope_check.py +0 -116
- package/src/debug/reference/hf_weights.py +0 -75
- package/src/errors/doppler-error.d.ts +0 -20
- package/src/experimental/bridge/extension/background.d.ts +0 -14
- package/src/experimental/bridge/extension/background.js +0 -168
- package/src/experimental/bridge/extension/manifest.json +0 -34
- package/src/experimental/bridge/extension-client.d.ts +0 -114
- package/src/experimental/bridge/index.d.ts +0 -69
- package/src/experimental/bridge/node-bridge.js +0 -24
- package/src/experimental/bridge/protocol.d.ts +0 -96
- package/src/experimental/browser/gguf-importer.d.ts +0 -136
- package/src/experimental/browser/gguf-importer.js +0 -532
- package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
- package/src/experimental/browser/quantization.d.ts +0 -74
- package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
- package/src/experimental/browser/shard-io-browser.d.ts +0 -57
- package/src/experimental/browser/shard-io-browser.js +0 -89
- package/src/experimental/browser/tensor-source-download.d.ts +0 -27
- package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
- package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
- package/src/experimental/distribution/p2p-transport.js +0 -75
- package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
- package/src/experimental/hotswap/manifest.d.ts +0 -42
- package/src/experimental/hotswap/model-swap.js +0 -42
- package/src/experimental/hotswap/runtime.d.ts +0 -31
- package/src/experimental/training/README.md +0 -164
- package/src/experimental/training/artifacts.d.ts +0 -160
- package/src/experimental/training/attention-backward.d.ts +0 -30
- package/src/experimental/training/attention-forward.d.ts +0 -22
- package/src/experimental/training/attention-forward.js +0 -82
- package/src/experimental/training/autograd.d.ts +0 -51
- package/src/experimental/training/backward-kernels.js +0 -39
- package/src/experimental/training/checkpoint-watch.d.ts +0 -8
- package/src/experimental/training/checkpoint.d.ts +0 -36
- package/src/experimental/training/clip.d.ts +0 -9
- package/src/experimental/training/dataloader.d.ts +0 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
- package/src/experimental/training/datasets/gepa-frontier.js +0 -220
- package/src/experimental/training/datasets/index.d.ts +0 -24
- package/src/experimental/training/datasets/index.js +0 -13
- package/src/experimental/training/datasets/jsonl.d.ts +0 -11
- package/src/experimental/training/datasets/reploid.d.ts +0 -3
- package/src/experimental/training/datasets/reploid.js +0 -36
- package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
- package/src/experimental/training/datasets/teacher-traces.js +0 -279
- package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
- package/src/experimental/training/datasets/token-batch.d.ts +0 -21
- package/src/experimental/training/datasets/token-batch.js +0 -52
- package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
- package/src/experimental/training/datasets/translation-pairs.js +0 -49
- package/src/experimental/training/distillation/artifacts.d.ts +0 -71
- package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
- package/src/experimental/training/distillation/dataset.d.ts +0 -59
- package/src/experimental/training/distillation/eval.d.ts +0 -34
- package/src/experimental/training/distillation/index.d.ts +0 -29
- package/src/experimental/training/distillation/runtime.d.ts +0 -20
- package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
- package/src/experimental/training/distillation/stage-a.d.ts +0 -45
- package/src/experimental/training/distillation/stage-b.d.ts +0 -24
- package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
- package/src/experimental/training/distillation/suite-data.d.ts +0 -45
- package/src/experimental/training/export.d.ts +0 -41
- package/src/experimental/training/index.d.ts +0 -74
- package/src/experimental/training/index.js +0 -63
- package/src/experimental/training/lora-pipeline.d.ts +0 -183
- package/src/experimental/training/lora.d.ts +0 -19
- package/src/experimental/training/loss-scaling.d.ts +0 -21
- package/src/experimental/training/loss.d.ts +0 -10
- package/src/experimental/training/objectives/base.d.ts +0 -58
- package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
- package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
- package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
- package/src/experimental/training/objectives/index.d.ts +0 -12
- package/src/experimental/training/objectives/index.js +0 -6
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
- package/src/experimental/training/operator-agent-eval.d.ts +0 -65
- package/src/experimental/training/operator-artifacts.d.ts +0 -62
- package/src/experimental/training/operator-command.d.ts +0 -5
- package/src/experimental/training/operator-eval.d.ts +0 -48
- package/src/experimental/training/operator-scoreboard.d.ts +0 -5
- package/src/experimental/training/optimizer.d.ts +0 -22
- package/src/experimental/training/runner.d.ts +0 -249
- package/src/experimental/training/suite.d.ts +0 -299
- package/src/experimental/training/tensor-factory.d.ts +0 -9
- package/src/experimental/training/trainer.d.ts +0 -89
- package/src/experimental/training/ul_dataset.d.ts +0 -47
- package/src/experimental/training/ul_schedule.d.ts +0 -6
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
- package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
- package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
- package/src/experimental/training/workload-packs/registry.json +0 -278
- package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
- package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
- package/src/experimental/training/workloads.d.ts +0 -201
- package/src/formats/gguf/types.d.ts +0 -74
- package/src/formats/index.d.ts +0 -75
- package/src/formats/index.js +0 -19
- package/src/formats/litert/types.d.ts +0 -74
- package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
- package/src/formats/rdrr/distributed/validation.d.ts +0 -23
- package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
- package/src/formats/safetensors/types.d.ts +0 -67
- package/src/formats/tflite/types.d.ts +0 -61
- package/src/formats/tokenizer/index.d.ts +0 -5
- package/src/formats/tokenizer/index.js +0 -3
- package/src/formats/tokenizer/types.d.ts +0 -9
- package/src/gpu/kernel-runtime.d.ts +0 -13
- package/src/gpu/kernel-selection-cache.d.ts +0 -13
- package/src/gpu/kernel-selection-log.d.ts +0 -4
- package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
- package/src/gpu/kernel-tuner/cache.d.ts +0 -55
- package/src/gpu/kernel-tuner/index.d.ts +0 -59
- package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
- package/src/gpu/kernel-tuner/types.d.ts +0 -101
- package/src/gpu/kernel-tuner/types.js +0 -4
- package/src/gpu/kernel-tuner.d.ts +0 -33
- package/src/gpu/kernels/README.md +0 -166
- package/src/gpu/kernels/check-finiteness.d.ts +0 -15
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
- package/src/gpu/kernels/check-stop.d.ts +0 -31
- package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
- package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
- package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
- package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
- package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
- package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
- package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
- package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
- package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
- package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
- package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
- package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
- package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
- package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
- package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
- package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
- package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
- package/src/gpu/kernels/constants.d.ts +0 -129
- package/src/gpu/kernels/dispatch.d.ts +0 -72
- package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
- package/src/gpu/kernels/kernel-base.d.ts +0 -33
- package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
- package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
- package/src/gpu/kernels/matmul-selection.d.ts +0 -87
- package/src/gpu/kernels/rep-penalty.d.ts +0 -21
- package/src/gpu/kernels/rule-matcher.d.ts +0 -30
- package/src/gpu/kernels/rule-registry.d.ts +0 -7
- package/src/gpu/kernels/types.js +0 -4
- package/src/gpu/perf-guards.d.ts +0 -13
- package/src/gpu/profiler.d.ts +0 -93
- package/src/gpu/readback-utils.d.ts +0 -16
- package/src/gpu/readback-utils.js +0 -49
- package/src/gpu/submit-probe.d.ts +0 -12
- package/src/index-browser.d.ts +0 -2
- package/src/index-browser.js +0 -2
- package/src/inference/README.md +0 -593
- package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
- package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
- package/src/inference/browser-harness-model-helpers.d.ts +0 -10
- package/src/inference/browser-harness-report-helpers.d.ts +0 -7
- package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
- package/src/inference/browser-harness-text-helpers.d.ts +0 -152
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
- package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
- package/src/inference/network-evolution.js +0 -79
- package/src/inference/pipelines/context.d.ts +0 -21
- package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
- package/src/inference/pipelines/diffusion/index.d.ts +0 -3
- package/src/inference/pipelines/diffusion/index.js +0 -3
- package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
- package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
- package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
- package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
- package/src/inference/pipelines/diffusion/types.js +0 -1
- package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
- package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
- package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
- package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
- package/src/inference/pipelines/dream/pipeline.js +0 -8
- package/src/inference/pipelines/energy/index.d.ts +0 -1
- package/src/inference/pipelines/energy/index.js +0 -1
- package/src/inference/pipelines/energy/types.js +0 -1
- package/src/inference/pipelines/energy-head/index.d.ts +0 -6
- package/src/inference/pipelines/energy-head/index.js +0 -6
- package/src/inference/pipelines/factory.d.ts +0 -10
- package/src/inference/pipelines/index.d.ts +0 -22
- package/src/inference/pipelines/index.js +0 -19
- package/src/inference/pipelines/registry.d.ts +0 -15
- package/src/inference/pipelines/rng.d.ts +0 -2
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
- package/src/inference/pipelines/text/attention/index.d.ts +0 -24
- package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
- package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
- package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
- package/src/inference/pipelines/text/attention/record.d.ts +0 -36
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
- package/src/inference/pipelines/text/attention/run.d.ts +0 -38
- package/src/inference/pipelines/text/attention/types.d.ts +0 -125
- package/src/inference/pipelines/text/attention.d.ts +0 -23
- package/src/inference/pipelines/text/attention.js +0 -12
- package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
- package/src/inference/pipelines/text/buffer-types.js +0 -4
- package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
- package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
- package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
- package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
- package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
- package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
- package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
- package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
- package/src/inference/pipelines/text/embed.d.ts +0 -69
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
- package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
- package/src/inference/pipelines/text/failure-reduction.js +0 -180
- package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
- package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
- package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
- package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
- package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
- package/src/inference/pipelines/text/ffn.d.ts +0 -31
- package/src/inference/pipelines/text/ffn.js +0 -18
- package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
- package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
- package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
- package/src/inference/pipelines/text/index.d.ts +0 -5
- package/src/inference/pipelines/text/index.js +0 -6
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
- package/src/inference/pipelines/text/layer.d.ts +0 -118
- package/src/inference/pipelines/text/load-errors.d.ts +0 -21
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
- package/src/inference/pipelines/text/logits/types.js +0 -4
- package/src/inference/pipelines/text/logits.d.ts +0 -27
- package/src/inference/pipelines/text/logits.js +0 -16
- package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
- package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
- package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
- package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
- package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
- package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
- package/src/inference/pipelines/text/operator-events.d.ts +0 -125
- package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
- package/src/inference/pipelines/text/ops.d.ts +0 -222
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
- package/src/inference/pipelines/text/probes.d.ts +0 -28
- package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
- package/src/inference/pipelines/text/stage-names.d.ts +0 -88
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
- package/src/inference/pipelines/text/types.js +0 -4
- package/src/inference/tokenizers/base.d.ts +0 -44
- package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
- package/src/inference/tokenizers/bpe.d.ts +0 -27
- package/src/inference/tokenizers/bundled.d.ts +0 -67
- package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
- package/src/inference/tokenizers/types.js +0 -7
- package/src/loader/embedding-limit-preflight.d.ts +0 -31
- package/src/loader/per-layer-input-loader.d.ts +0 -30
- package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
- package/src/loader/tensors/source-transform.d.ts +0 -65
- package/src/memory/address-table.d.ts +0 -96
- package/src/rules/execution-rules-contract-check.d.ts +0 -21
- package/src/rules/layer-pattern-contract-check.d.ts +0 -17
- package/src/rules/rule-registry.d.ts +0 -77
- package/src/storage/artifact-storage-context.d.ts +0 -63
- package/src/storage/backends/idb-store.d.ts +0 -52
- package/src/storage/backends/memory-store.d.ts +0 -36
- package/src/storage/backends/opfs-store.d.ts +0 -41
- package/src/storage/blake3.d.ts +0 -17
- package/src/storage/download-types.d.ts +0 -157
- package/src/storage/downloader.d.ts +0 -103
- package/src/storage/index.d.ts +0 -274
- package/src/storage/index.js +0 -201
- package/src/storage/preflight.d.ts +0 -144
- package/src/storage/preflight.js +0 -360
- package/src/storage/quickstart-downloader.d.ts +0 -157
- package/src/storage/quickstart-downloader.js +0 -289
- package/src/storage/source-artifact-store.d.ts +0 -59
- package/src/sw.js +0 -184
- package/src/tooling/command-api-constants.d.ts +0 -9
- package/src/tooling/command-api-family-normalizers.d.ts +0 -10
- package/src/tooling/command-api-helpers.d.ts +0 -23
- package/src/tooling/command-runner-shared.d.ts +0 -89
- package/src/tooling/conversion-config-materializer.d.ts +0 -14
- package/src/tooling/conversion-config-materializer.js +0 -96
- package/src/tooling/diagnose-runner.d.ts +0 -28
- package/src/tooling/hf-registry-utils.d.ts +0 -55
- package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
- package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
- package/src/tooling/kernel-path-builder/index.d.ts +0 -65
- package/src/tooling/kernel-path-builder/index.js +0 -1361
- package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
- package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
- package/src/tooling/lean-execution-contract-runner.js +0 -158
- package/src/tooling/lean-execution-contract.d.ts +0 -16
- package/src/tooling/lean-execution-contract.js +0 -81
- package/src/tooling/litert-package-runtime.d.ts +0 -46
- package/src/tooling/node-convert-worker-pool.d.ts +0 -16
- package/src/tooling/node-convert-worker.d.ts +0 -1
- package/src/tooling/node-converter.d.ts +0 -1
- package/src/tooling/node-file-fetch.d.ts +0 -1
- package/src/tooling/node-source-runtime.d.ts +0 -24
- package/src/tooling/node-webgpu.d.ts +0 -17
- package/src/tooling/precision-replay-math.js +0 -174
- package/src/tooling/program-bundle-host.d.ts +0 -16
- package/src/tooling/program-bundle-host.js +0 -20
- package/src/tooling/runtime-input-composition.d.ts +0 -37
- package/src/tooling/source-package-profiles.d.ts +0 -50
- package/src/tooling/source-runtime-converter-config.d.ts +0 -19
- package/src/tooling/source-runtime-materializer.d.ts +0 -6
- package/src/tooling/source-runtime-materializer.js +0 -89
- package/src/tooling-exports.browser.d.ts +0 -7
- package/src/types/chrome.d.ts +0 -36
- package/src/types/chrome.js +0 -1
- package/src/types/index.d.ts +0 -1
- package/src/types/index.js +0 -1
- package/src/utils/array-buffer.d.ts +0 -4
- package/src/utils/canonical-hash.d.ts +0 -7
- package/src/utils/clone-json.d.ts +0 -1
- package/src/utils/hf-resolve-url.d.ts +0 -16
- package/src/utils/index.d.ts +0 -10
- package/src/utils/index.js +0 -10
- package/src/utils/load-json.d.ts +0 -5
- package/src/utils/plain-object.d.ts +0 -1
- package/src/utils/sha256.d.ts +0 -5
- package/src/utils/stable-sort-object.d.ts +0 -1
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import type { CommandRecorder, ProfileTimings } from '../../../gpu/command-recorder.js';
|
|
11
11
|
import type { PipelineState } from './state.js';
|
|
12
|
-
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerContext } from './types.js';
|
|
12
|
+
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerContext, WorkloadPhaseTiming } from './types.js';
|
|
13
13
|
import type { LogitsConfig, LogitsWeights } from './logits/index.js';
|
|
14
14
|
import type { WeightBufferConfig } from './weights.js';
|
|
15
15
|
import type { ChatMessage } from './chat-format.js';
|
|
@@ -75,6 +75,23 @@ export declare class PipelineGenerator {
|
|
|
75
75
|
): Promise<DiffusionGemmaCanvasStepResult>;
|
|
76
76
|
prefillWithEmbedding(prompt: PromptInput, options?: GenerateOptions): Promise<PrefillEmbeddingResult>;
|
|
77
77
|
prefillWithLogits(prompt: PromptInput, options?: GenerateOptions): Promise<PrefillResult>;
|
|
78
|
+
prefillWithTokenLogits(prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{
|
|
79
|
+
seqLen: number;
|
|
80
|
+
tokens: number[];
|
|
81
|
+
tokenIds: number[];
|
|
82
|
+
logits: Float32Array;
|
|
83
|
+
logitsByTokenId: Record<number, number>;
|
|
84
|
+
phase?: WorkloadPhaseTiming | null;
|
|
85
|
+
}>;
|
|
86
|
+
prefillWithTokenLogitsFromKV(prefix: KVCacheSnapshot, prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{
|
|
87
|
+
seqLen: number;
|
|
88
|
+
prefixTokens: number[];
|
|
89
|
+
tokens: number[];
|
|
90
|
+
tokenIds: number[];
|
|
91
|
+
logits: Float32Array;
|
|
92
|
+
logitsByTokenId: Record<number, number>;
|
|
93
|
+
phase?: WorkloadPhaseTiming | null;
|
|
94
|
+
}>;
|
|
78
95
|
decodeStepLogits(currentIds: number[], options?: GenerateOptions): Promise<LogitsStepResult>;
|
|
79
96
|
advanceWithToken(tokenId: number, options?: GenerateOptions): Promise<void>;
|
|
80
97
|
advanceWithTokenAndEmbedding(tokenId: number, options?: GenerateOptions): Promise<AdvanceEmbeddingResult>;
|
|
@@ -134,6 +134,36 @@ const FINITENESS_RESET_WORDS = new Uint32Array(4);
|
|
|
134
134
|
const tokenizerSuppressionCache = new WeakMap();
|
|
135
135
|
const PREFILL_CHUNK_SUBMIT_MODES = new Set(['sync', 'async']);
|
|
136
136
|
|
|
137
|
+
function mergeRecordOpLabelCounts(target, source) {
|
|
138
|
+
const merged = target && typeof target === 'object' && !Array.isArray(target)
|
|
139
|
+
? target
|
|
140
|
+
: {};
|
|
141
|
+
if (!source || typeof source !== 'object' || Array.isArray(source)) {
|
|
142
|
+
return merged;
|
|
143
|
+
}
|
|
144
|
+
for (const [label, rawCount] of Object.entries(source)) {
|
|
145
|
+
const count = Number(rawCount);
|
|
146
|
+
if (typeof label !== 'string' || label.length === 0 || !Number.isFinite(count) || count <= 0) {
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
merged[label] = (Number.isFinite(merged[label]) ? merged[label] : 0) + count;
|
|
150
|
+
}
|
|
151
|
+
return merged;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function recordPrefillRecorderStats(state, recorder) {
|
|
155
|
+
if (!recorder || typeof recorder.getStats !== 'function') {
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
const recordStats = recorder.getStats();
|
|
159
|
+
state.stats.prefillRecordOps = (state.stats.prefillRecordOps ?? 0) + recordStats.opCount;
|
|
160
|
+
state.stats.prefillRecordPasses = (state.stats.prefillRecordPasses ?? 0) + recordStats.computePassCount;
|
|
161
|
+
state.stats.prefillRecordOpLabels = mergeRecordOpLabelCounts(
|
|
162
|
+
state.stats.prefillRecordOpLabels,
|
|
163
|
+
recordStats.opLabelCounts
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
|
|
137
167
|
export function resolvePrefillChunkSubmitMode(runtimeConfig, modelConfig) {
|
|
138
168
|
const runtimeSubmit = runtimeConfig?.inference?.session?.prefillChunkSubmitMode;
|
|
139
169
|
const manifestSubmit = modelConfig?.sessionSettings?.prefillChunkSubmitMode;
|
|
@@ -219,6 +249,26 @@ function resolveSuppressedSamplingTokenIds(state, samplingConfig) {
|
|
|
219
249
|
return [...suppressed];
|
|
220
250
|
}
|
|
221
251
|
|
|
252
|
+
function normalizeSelectedLogitTokenIds(value, vocabSize, label) {
|
|
253
|
+
if (value == null) {
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
if (!Array.isArray(value) && !ArrayBuffer.isView(value)) {
|
|
257
|
+
throw new Error(`${label} must be an array or typed array.`);
|
|
258
|
+
}
|
|
259
|
+
const tokenIds = Array.from(value, (entry, index) => {
|
|
260
|
+
const tokenId = Number(entry);
|
|
261
|
+
if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSize) {
|
|
262
|
+
throw new Error(`${label}[${index}] must be an integer in [0, ${vocabSize}), got "${String(entry)}".`);
|
|
263
|
+
}
|
|
264
|
+
return tokenId;
|
|
265
|
+
});
|
|
266
|
+
if (tokenIds.length === 0) {
|
|
267
|
+
throw new Error(`${label} must not be empty.`);
|
|
268
|
+
}
|
|
269
|
+
return tokenIds;
|
|
270
|
+
}
|
|
271
|
+
|
|
222
272
|
async function traceActivationHealth(label, buffer, dtype, elementCount) {
|
|
223
273
|
if (!isTraceEnabled('logits') || !isGpuBufferInstance(buffer)) {
|
|
224
274
|
return;
|
|
@@ -535,6 +585,11 @@ export class PipelineGenerator {
|
|
|
535
585
|
this.#state.stats.decodeProfileSteps = [];
|
|
536
586
|
this.#state.stats.ttftMs = 0;
|
|
537
587
|
this.#state.stats.decodeTimeMs = 0;
|
|
588
|
+
this.#state.stats.prefillRecordMs = 0;
|
|
589
|
+
this.#state.stats.prefillRecordOps = 0;
|
|
590
|
+
this.#state.stats.prefillRecordPasses = 0;
|
|
591
|
+
this.#state.stats.prefillRecordOpLabels = {};
|
|
592
|
+
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
538
593
|
this.#state.stats.decodeRecordMs = 0;
|
|
539
594
|
this.#state.stats.decodeRecordOps = 0;
|
|
540
595
|
this.#state.stats.decodeRecordPasses = 0;
|
|
@@ -667,6 +722,7 @@ export class PipelineGenerator {
|
|
|
667
722
|
|
|
668
723
|
try {
|
|
669
724
|
if (currentRecorder) {
|
|
725
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
670
726
|
await currentRecorder.submitAndWait();
|
|
671
727
|
await recordProfile(currentRecorder);
|
|
672
728
|
} else {
|
|
@@ -703,12 +759,15 @@ export class PipelineGenerator {
|
|
|
703
759
|
|
|
704
760
|
async _prefillPromptToLogits(prompt, opts, contextLabel) {
|
|
705
761
|
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
762
|
+
const inputStart = performance.now();
|
|
706
763
|
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, contextLabel, opts.inputIds);
|
|
764
|
+
const inputMs = performance.now() - inputStart;
|
|
707
765
|
if (opts.debug) {
|
|
708
766
|
log.debug('Pipeline', `${contextLabel}: ${inputIds.length} tokens`);
|
|
709
767
|
}
|
|
710
768
|
|
|
711
769
|
let logits;
|
|
770
|
+
const prefillStart = performance.now();
|
|
712
771
|
const runPrefill = () => this._prefillInputIdsToLogits(inputIds, opts);
|
|
713
772
|
try {
|
|
714
773
|
logits = await runPrefill();
|
|
@@ -726,7 +785,21 @@ export class PipelineGenerator {
|
|
|
726
785
|
);
|
|
727
786
|
}
|
|
728
787
|
|
|
729
|
-
return {
|
|
788
|
+
return {
|
|
789
|
+
inputIds,
|
|
790
|
+
logits,
|
|
791
|
+
phase: {
|
|
792
|
+
inputMs,
|
|
793
|
+
prefillMs: performance.now() - prefillStart,
|
|
794
|
+
tokens: inputIds.length,
|
|
795
|
+
prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
|
|
796
|
+
prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
|
|
797
|
+
prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
|
|
798
|
+
prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
|
|
799
|
+
prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
|
|
800
|
+
gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
|
|
801
|
+
},
|
|
802
|
+
};
|
|
730
803
|
}
|
|
731
804
|
|
|
732
805
|
async _decodeStepToLogits(currentIds, opts) {
|
|
@@ -785,6 +858,9 @@ export class PipelineGenerator {
|
|
|
785
858
|
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
786
859
|
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
787
860
|
this.#state.stats.prefillRecordMs = 0;
|
|
861
|
+
this.#state.stats.prefillRecordOps = 0;
|
|
862
|
+
this.#state.stats.prefillRecordPasses = 0;
|
|
863
|
+
this.#state.stats.prefillRecordOpLabels = {};
|
|
788
864
|
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
789
865
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
790
866
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
@@ -1591,6 +1667,7 @@ export class PipelineGenerator {
|
|
|
1591
1667
|
} = prefillResult;
|
|
1592
1668
|
currentHiddenBuffer = prefillResult.currentHiddenBuffer;
|
|
1593
1669
|
if (currentRecorder) {
|
|
1670
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
1594
1671
|
await currentRecorder.submitAndWait();
|
|
1595
1672
|
await recordProfile(currentRecorder);
|
|
1596
1673
|
} else {
|
|
@@ -1678,6 +1755,7 @@ export class PipelineGenerator {
|
|
|
1678
1755
|
} = prefillResult;
|
|
1679
1756
|
currentHiddenBuffer = prefillResult.currentHiddenBuffer;
|
|
1680
1757
|
if (currentRecorder) {
|
|
1758
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
1681
1759
|
await currentRecorder.submitAndWait();
|
|
1682
1760
|
await recordProfile(currentRecorder);
|
|
1683
1761
|
} else {
|
|
@@ -1810,6 +1888,9 @@ export class PipelineGenerator {
|
|
|
1810
1888
|
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
1811
1889
|
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
1812
1890
|
this.#state.stats.prefillRecordMs = 0;
|
|
1891
|
+
this.#state.stats.prefillRecordOps = 0;
|
|
1892
|
+
this.#state.stats.prefillRecordPasses = 0;
|
|
1893
|
+
this.#state.stats.prefillRecordOpLabels = {};
|
|
1813
1894
|
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
1814
1895
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
1815
1896
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
@@ -1991,6 +2072,7 @@ export class PipelineGenerator {
|
|
|
1991
2072
|
|
|
1992
2073
|
// Ensure prefill work completes before returning a usable snapshot.
|
|
1993
2074
|
if (currentRecorder) {
|
|
2075
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
1994
2076
|
await currentRecorder.submitAndWait();
|
|
1995
2077
|
await recordProfile(currentRecorder);
|
|
1996
2078
|
} else {
|
|
@@ -2025,20 +2107,27 @@ export class PipelineGenerator {
|
|
|
2025
2107
|
throw new Error('Generation already in progress');
|
|
2026
2108
|
}
|
|
2027
2109
|
assertIncrementalDecodeSupport(this.#state, 'prefillWithEmbedding');
|
|
2110
|
+
const totalStart = performance.now();
|
|
2028
2111
|
this._resetDecodeRuntimeState();
|
|
2029
2112
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2030
2113
|
this.#state.stats.prefillProfileSteps = [];
|
|
2031
2114
|
const opts = resolvePrefillEmbeddingOptions(this.#state, options);
|
|
2032
2115
|
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
2116
|
+
const inputStart = performance.now();
|
|
2033
2117
|
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
|
|
2118
|
+
const inputMs = performance.now() - inputStart;
|
|
2034
2119
|
if (opts.debug) {
|
|
2035
2120
|
log.debug('Pipeline', `PrefillWithEmbedding: ${inputIds.length} tokens (mode=${opts.embeddingMode})`);
|
|
2036
2121
|
}
|
|
2037
2122
|
|
|
2038
2123
|
try {
|
|
2039
2124
|
let prefillResult;
|
|
2125
|
+
const prefillStart = performance.now();
|
|
2040
2126
|
try {
|
|
2041
|
-
prefillResult = await this._prefillToHidden(inputIds,
|
|
2127
|
+
prefillResult = await this._prefillToHidden(inputIds, {
|
|
2128
|
+
...opts,
|
|
2129
|
+
_embeddingOnly: options.__skipStateSnapshot === true,
|
|
2130
|
+
});
|
|
2042
2131
|
} catch (error) {
|
|
2043
2132
|
if (shouldRetryWithFinitenessFallback(error)) {
|
|
2044
2133
|
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillWithEmbedding. Retrying with F32 precision.`);
|
|
@@ -2046,13 +2135,17 @@ export class PipelineGenerator {
|
|
|
2046
2135
|
opts,
|
|
2047
2136
|
'prefillWithEmbedding',
|
|
2048
2137
|
1,
|
|
2049
|
-
() => this._prefillToHidden(inputIds,
|
|
2138
|
+
() => this._prefillToHidden(inputIds, {
|
|
2139
|
+
...opts,
|
|
2140
|
+
_embeddingOnly: options.__skipStateSnapshot === true,
|
|
2141
|
+
}),
|
|
2050
2142
|
prefillStartSeqLen
|
|
2051
2143
|
);
|
|
2052
2144
|
} else {
|
|
2053
2145
|
throw error;
|
|
2054
2146
|
}
|
|
2055
2147
|
}
|
|
2148
|
+
const prefillMs = performance.now() - prefillStart;
|
|
2056
2149
|
|
|
2057
2150
|
const {
|
|
2058
2151
|
numTokens,
|
|
@@ -2066,7 +2159,10 @@ export class PipelineGenerator {
|
|
|
2066
2159
|
} = prefillResult;
|
|
2067
2160
|
|
|
2068
2161
|
// Ensure prefill work completes before readback.
|
|
2162
|
+
let submitWaitMs = 0;
|
|
2163
|
+
const submitWaitStart = performance.now();
|
|
2069
2164
|
if (currentRecorder) {
|
|
2165
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
2070
2166
|
await currentRecorder.submitAndWait();
|
|
2071
2167
|
await recordProfile(currentRecorder);
|
|
2072
2168
|
} else {
|
|
@@ -2075,21 +2171,34 @@ export class PipelineGenerator {
|
|
|
2075
2171
|
await device.queue.onSubmittedWorkDone();
|
|
2076
2172
|
}
|
|
2077
2173
|
}
|
|
2174
|
+
submitWaitMs = performance.now() - submitWaitStart;
|
|
2078
2175
|
|
|
2079
2176
|
if (!allowReadback('pipeline.prefill.embedding')) {
|
|
2080
2177
|
throw new Error('GPU readback disabled; cannot return embedding');
|
|
2081
2178
|
}
|
|
2082
2179
|
|
|
2083
2180
|
let embedding;
|
|
2181
|
+
let hiddenBytes = 0;
|
|
2182
|
+
let readbackMs = 0;
|
|
2183
|
+
let decodeHiddenMs = 0;
|
|
2184
|
+
let finalNormMs = 0;
|
|
2185
|
+
let extractMs = 0;
|
|
2084
2186
|
try {
|
|
2085
2187
|
const hiddenSize = config.hiddenSize;
|
|
2086
|
-
|
|
2188
|
+
hiddenBytes = numTokens * hiddenSize * activationBytes;
|
|
2189
|
+
const readbackStart = performance.now();
|
|
2087
2190
|
const hiddenData = await readBuffer(currentHiddenBuffer, hiddenBytes);
|
|
2191
|
+
readbackMs = performance.now() - readbackStart;
|
|
2088
2192
|
if (hiddenData.byteLength === 0) {
|
|
2089
2193
|
throw new Error('GPU readback disabled; cannot return embedding');
|
|
2090
2194
|
}
|
|
2195
|
+
const decodeStart = performance.now();
|
|
2091
2196
|
const hiddenStates = decodeReadback(hiddenData, activationDtype);
|
|
2197
|
+
decodeHiddenMs = performance.now() - decodeStart;
|
|
2198
|
+
const finalNormStart = performance.now();
|
|
2092
2199
|
const finalNormWeights = await this._getFinalNormWeights();
|
|
2200
|
+
finalNormMs = performance.now() - finalNormStart;
|
|
2201
|
+
const extractStart = performance.now();
|
|
2093
2202
|
embedding = this._extractEmbeddingFromHidden(
|
|
2094
2203
|
hiddenStates,
|
|
2095
2204
|
numTokens,
|
|
@@ -2098,11 +2207,31 @@ export class PipelineGenerator {
|
|
|
2098
2207
|
finalNormWeights,
|
|
2099
2208
|
config
|
|
2100
2209
|
);
|
|
2210
|
+
extractMs = performance.now() - extractStart;
|
|
2101
2211
|
} finally {
|
|
2102
2212
|
releaseBuffer(currentHiddenBuffer);
|
|
2103
2213
|
}
|
|
2104
2214
|
|
|
2105
2215
|
this.#state.currentSeqLen = startPos + numTokens;
|
|
2216
|
+
const phase = {
|
|
2217
|
+
totalMs: performance.now() - totalStart,
|
|
2218
|
+
inputMs,
|
|
2219
|
+
prefillMs,
|
|
2220
|
+
submitWaitMs,
|
|
2221
|
+
readbackMs,
|
|
2222
|
+
decodeHiddenMs,
|
|
2223
|
+
finalNormMs,
|
|
2224
|
+
extractMs,
|
|
2225
|
+
hiddenBytes,
|
|
2226
|
+
tokens: numTokens,
|
|
2227
|
+
activationDtype,
|
|
2228
|
+
prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
|
|
2229
|
+
prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
|
|
2230
|
+
prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
|
|
2231
|
+
prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
|
|
2232
|
+
prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
|
|
2233
|
+
gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
|
|
2234
|
+
};
|
|
2106
2235
|
|
|
2107
2236
|
// Batch embedding skips expensive KV cache clone and linear attention clone
|
|
2108
2237
|
// since the caller will reset immediately after extracting the embedding.
|
|
@@ -2113,6 +2242,7 @@ export class PipelineGenerator {
|
|
|
2113
2242
|
tokens: inputIds,
|
|
2114
2243
|
embedding,
|
|
2115
2244
|
embeddingMode: opts.embeddingMode,
|
|
2245
|
+
phase,
|
|
2116
2246
|
linearAttention: null,
|
|
2117
2247
|
};
|
|
2118
2248
|
}
|
|
@@ -2128,6 +2258,7 @@ export class PipelineGenerator {
|
|
|
2128
2258
|
tokens: inputIds,
|
|
2129
2259
|
embedding,
|
|
2130
2260
|
embeddingMode: opts.embeddingMode,
|
|
2261
|
+
phase,
|
|
2131
2262
|
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
2132
2263
|
};
|
|
2133
2264
|
} finally {
|
|
@@ -2141,12 +2272,34 @@ export class PipelineGenerator {
|
|
|
2141
2272
|
throw new Error('Generation already in progress');
|
|
2142
2273
|
}
|
|
2143
2274
|
assertIncrementalDecodeSupport(this.#state, 'prefillWithLogits');
|
|
2275
|
+
const totalStart = performance.now();
|
|
2144
2276
|
this._resetDecodeRuntimeState();
|
|
2145
2277
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2146
2278
|
this.#state.stats.prefillProfileSteps = [];
|
|
2147
2279
|
const opts = resolvePrefillOptions(this.#state, options);
|
|
2280
|
+
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
2281
|
+
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
2282
|
+
if (diagnosticsEnabled) {
|
|
2283
|
+
const captureConfig = {
|
|
2284
|
+
...createDefaultCaptureConfig(),
|
|
2285
|
+
enabled: true,
|
|
2286
|
+
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
2287
|
+
...(options?.diagnostics?.captureConfig ?? {}),
|
|
2288
|
+
};
|
|
2289
|
+
validateCaptureConfig(captureConfig);
|
|
2290
|
+
this.#state.operatorDiagnostics = {
|
|
2291
|
+
enabled: true,
|
|
2292
|
+
captureConfig,
|
|
2293
|
+
emitter: new OperatorEventEmitter({
|
|
2294
|
+
modelHash: this.#state.manifest?.modelId ?? null,
|
|
2295
|
+
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
2296
|
+
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
2297
|
+
}),
|
|
2298
|
+
tsirFixture: null,
|
|
2299
|
+
};
|
|
2300
|
+
}
|
|
2148
2301
|
try {
|
|
2149
|
-
const { inputIds, logits } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
|
|
2302
|
+
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
|
|
2150
2303
|
|
|
2151
2304
|
const snapshot = this.#state.kvCache?.clone();
|
|
2152
2305
|
if (!snapshot) {
|
|
@@ -2158,8 +2311,134 @@ export class PipelineGenerator {
|
|
|
2158
2311
|
seqLen: this.#state.currentSeqLen,
|
|
2159
2312
|
tokens: inputIds,
|
|
2160
2313
|
logits,
|
|
2314
|
+
phase: {
|
|
2315
|
+
...(prefillPhase ?? {}),
|
|
2316
|
+
totalMs: performance.now() - totalStart,
|
|
2317
|
+
},
|
|
2161
2318
|
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
2162
2319
|
};
|
|
2320
|
+
} finally {
|
|
2321
|
+
this._closeFinitenessFallbackWindow(opts);
|
|
2322
|
+
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
2323
|
+
? {
|
|
2324
|
+
enabled: true,
|
|
2325
|
+
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
2326
|
+
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
2327
|
+
}
|
|
2328
|
+
: null;
|
|
2329
|
+
this.#state.operatorDiagnostics = null;
|
|
2330
|
+
}
|
|
2331
|
+
}
|
|
2332
|
+
|
|
2333
|
+
async prefillWithTokenLogits(prompt, tokenIds, options = {}) {
|
|
2334
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
2335
|
+
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2336
|
+
throw new Error('Generation already in progress');
|
|
2337
|
+
}
|
|
2338
|
+
assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogits');
|
|
2339
|
+
const totalStart = performance.now();
|
|
2340
|
+
const selectedTokenIds = normalizeSelectedLogitTokenIds(
|
|
2341
|
+
tokenIds,
|
|
2342
|
+
this.#state.modelConfig.vocabSize,
|
|
2343
|
+
'prefillWithTokenLogits.tokenIds'
|
|
2344
|
+
);
|
|
2345
|
+
this._resetDecodeRuntimeState();
|
|
2346
|
+
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2347
|
+
this.#state.stats.prefillProfileSteps = [];
|
|
2348
|
+
const opts = resolvePrefillOptions(this.#state, options);
|
|
2349
|
+
opts._selectedLogitTokenIds = selectedTokenIds;
|
|
2350
|
+
try {
|
|
2351
|
+
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogits');
|
|
2352
|
+
const logitsByTokenId = {};
|
|
2353
|
+
for (let index = 0; index < selectedTokenIds.length; index += 1) {
|
|
2354
|
+
logitsByTokenId[selectedTokenIds[index]] = logits[index];
|
|
2355
|
+
}
|
|
2356
|
+
return {
|
|
2357
|
+
seqLen: this.#state.currentSeqLen,
|
|
2358
|
+
tokens: inputIds,
|
|
2359
|
+
tokenIds: selectedTokenIds,
|
|
2360
|
+
logits,
|
|
2361
|
+
logitsByTokenId,
|
|
2362
|
+
phase: {
|
|
2363
|
+
...(prefillPhase ?? {}),
|
|
2364
|
+
totalMs: performance.now() - totalStart,
|
|
2365
|
+
selectedTokenCount: selectedTokenIds.length,
|
|
2366
|
+
},
|
|
2367
|
+
};
|
|
2368
|
+
} finally {
|
|
2369
|
+
this._closeFinitenessFallbackWindow(opts);
|
|
2370
|
+
}
|
|
2371
|
+
}
|
|
2372
|
+
|
|
2373
|
+
async prefillWithTokenLogitsFromKV(prefix, prompt, tokenIds, options = {}) {
|
|
2374
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
2375
|
+
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2376
|
+
throw new Error('Generation already in progress');
|
|
2377
|
+
}
|
|
2378
|
+
assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogitsFromKV');
|
|
2379
|
+
const totalStart = performance.now();
|
|
2380
|
+
if (!prefix || typeof prefix !== 'object' || !prefix.cache) {
|
|
2381
|
+
throw new Error('prefillWithTokenLogitsFromKV.prefix must be a KV cache snapshot.');
|
|
2382
|
+
}
|
|
2383
|
+
if (!Number.isInteger(prefix.seqLen) || prefix.seqLen < 0) {
|
|
2384
|
+
throw new Error('prefillWithTokenLogitsFromKV.prefix.seqLen must be a non-negative integer.');
|
|
2385
|
+
}
|
|
2386
|
+
if (!Array.isArray(prefix.tokens)) {
|
|
2387
|
+
throw new Error('prefillWithTokenLogitsFromKV.prefix.tokens must be an array.');
|
|
2388
|
+
}
|
|
2389
|
+
const selectedTokenIds = normalizeSelectedLogitTokenIds(
|
|
2390
|
+
tokenIds,
|
|
2391
|
+
this.#state.modelConfig.vocabSize,
|
|
2392
|
+
'prefillWithTokenLogitsFromKV.tokenIds'
|
|
2393
|
+
);
|
|
2394
|
+
|
|
2395
|
+
this.#state.kvCache = prefix.cache.clone();
|
|
2396
|
+
if (this.#state.useGPU && this.#state.kvCache) {
|
|
2397
|
+
const device = getDevice();
|
|
2398
|
+
if (device) {
|
|
2399
|
+
this.#state.kvCache.setGPUContext({ device });
|
|
2400
|
+
}
|
|
2401
|
+
}
|
|
2402
|
+
if (
|
|
2403
|
+
hasLinearAttentionLayers(this.#state.modelConfig.layerTypes)
|
|
2404
|
+
&& prefix.linearAttention == null
|
|
2405
|
+
) {
|
|
2406
|
+
throw new Error(
|
|
2407
|
+
'Prefix snapshot is missing linear_attention recurrent state. ' +
|
|
2408
|
+
'Regenerate the prefix snapshot using the current runtime.'
|
|
2409
|
+
);
|
|
2410
|
+
}
|
|
2411
|
+
this.#state.linearAttentionRuntime = restoreLinearAttentionRuntime(
|
|
2412
|
+
this.#state.linearAttentionRuntime,
|
|
2413
|
+
prefix.linearAttention ?? null
|
|
2414
|
+
);
|
|
2415
|
+
this.#state.currentSeqLen = prefix.seqLen;
|
|
2416
|
+
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2417
|
+
this.#state.stats.prefillProfileSteps = [];
|
|
2418
|
+
this.#state.decodeRing?.reset();
|
|
2419
|
+
|
|
2420
|
+
const opts = resolvePrefillOptions(this.#state, options);
|
|
2421
|
+
opts._selectedLogitTokenIds = selectedTokenIds;
|
|
2422
|
+
try {
|
|
2423
|
+
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogitsFromKV');
|
|
2424
|
+
const logitsByTokenId = {};
|
|
2425
|
+
for (let index = 0; index < selectedTokenIds.length; index += 1) {
|
|
2426
|
+
logitsByTokenId[selectedTokenIds[index]] = logits[index];
|
|
2427
|
+
}
|
|
2428
|
+
return {
|
|
2429
|
+
seqLen: this.#state.currentSeqLen,
|
|
2430
|
+
prefixTokens: prefix.tokens,
|
|
2431
|
+
tokens: inputIds,
|
|
2432
|
+
tokenIds: selectedTokenIds,
|
|
2433
|
+
logits,
|
|
2434
|
+
logitsByTokenId,
|
|
2435
|
+
phase: {
|
|
2436
|
+
...(prefillPhase ?? {}),
|
|
2437
|
+
totalMs: performance.now() - totalStart,
|
|
2438
|
+
selectedTokenCount: selectedTokenIds.length,
|
|
2439
|
+
prefixTokens: prefix.tokens.length,
|
|
2440
|
+
},
|
|
2441
|
+
};
|
|
2163
2442
|
} finally {
|
|
2164
2443
|
this._closeFinitenessFallbackWindow(opts);
|
|
2165
2444
|
}
|
|
@@ -2212,6 +2491,11 @@ export class PipelineGenerator {
|
|
|
2212
2491
|
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
2213
2492
|
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
2214
2493
|
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
2494
|
+
this.#state.stats.prefillRecordMs = 0;
|
|
2495
|
+
this.#state.stats.prefillRecordOps = 0;
|
|
2496
|
+
this.#state.stats.prefillRecordPasses = 0;
|
|
2497
|
+
this.#state.stats.prefillRecordOpLabels = {};
|
|
2498
|
+
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
2215
2499
|
this.#state.stats.ttftMs = 0;
|
|
2216
2500
|
const startTime = performance.now();
|
|
2217
2501
|
|
|
@@ -2412,6 +2696,7 @@ export class PipelineGenerator {
|
|
|
2412
2696
|
maxDecodeTokens: opts.maxTokens,
|
|
2413
2697
|
numLayers: this.#state.modelConfig.numLayers,
|
|
2414
2698
|
hiddenSize: this.#state.modelConfig.hiddenSize,
|
|
2699
|
+
configuredMaxBatchDecodeTokens: executionPlan.maxBatchDecodeTokens,
|
|
2415
2700
|
});
|
|
2416
2701
|
const requestedBatchTokens = executionPlan.batchSize * intervalBatches;
|
|
2417
2702
|
const boundedBatchTokens = maxBatchDecodeTokens == null
|
|
@@ -2725,6 +3010,11 @@ export class PipelineGenerator {
|
|
|
2725
3010
|
async _prefill(inputIds, opts) {
|
|
2726
3011
|
const numTokens = inputIds.length;
|
|
2727
3012
|
const config = this.#state.modelConfig;
|
|
3013
|
+
const selectedLogitTokenIds = normalizeSelectedLogitTokenIds(
|
|
3014
|
+
opts?._selectedLogitTokenIds,
|
|
3015
|
+
config.vocabSize,
|
|
3016
|
+
'_prefill.selectedLogitTokenIds'
|
|
3017
|
+
);
|
|
2728
3018
|
const startPos = this.#state.currentSeqLen;
|
|
2729
3019
|
const tracePrefillEnabled = isTraceEnabled('perf');
|
|
2730
3020
|
const prefillTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
@@ -2787,7 +3077,7 @@ export class PipelineGenerator {
|
|
|
2787
3077
|
if (!device || disableCommandBatching) return undefined;
|
|
2788
3078
|
return opts.profile
|
|
2789
3079
|
? createProfilingRecorder(label, device)
|
|
2790
|
-
: createCommandRecorder(label, { recordLabels: opts.debug === true }, device);
|
|
3080
|
+
: createCommandRecorder(label, { recordLabels: opts.debug === true || opts.benchmark === true }, device);
|
|
2791
3081
|
};
|
|
2792
3082
|
const recorder = createRecorder('prefill');
|
|
2793
3083
|
const debugCheckBuffer = this.#state.debug
|
|
@@ -2804,6 +3094,9 @@ export class PipelineGenerator {
|
|
|
2804
3094
|
);
|
|
2805
3095
|
context.currentTokenIds = inputIds;
|
|
2806
3096
|
context.diffusionGemmaDecoder = opts?._diffusionGemmaDecoder === true;
|
|
3097
|
+
context.skipKVCacheWrites = returnHidden
|
|
3098
|
+
&& opts?._embeddingOnly === true
|
|
3099
|
+
&& this.#state.runtimeConfig?.inference?.session?.skipEmbeddingKVCacheWrites === true;
|
|
2807
3100
|
context.multimodalBidirectionalSpan = multimodalBidirectionalSpan == null
|
|
2808
3101
|
? null
|
|
2809
3102
|
: {
|
|
@@ -2938,6 +3231,7 @@ export class PipelineGenerator {
|
|
|
2938
3231
|
hiddenStates.shape,
|
|
2939
3232
|
hiddenStates.label
|
|
2940
3233
|
);
|
|
3234
|
+
recordPrefillRecorderStats(this.#state, recorder);
|
|
2941
3235
|
await recorder.submitAndWait();
|
|
2942
3236
|
await recordProfile(recorder);
|
|
2943
3237
|
}
|
|
@@ -2972,6 +3266,7 @@ export class PipelineGenerator {
|
|
|
2972
3266
|
// buffers, preventing unbounded memory growth during large prefills. Critical for
|
|
2973
3267
|
// replay_prefill models where each decode step re-runs a prefill-style layer pass.
|
|
2974
3268
|
const prefillRecorderChunkLayers = resolvePrefillRecorderChunkLayers({
|
|
3269
|
+
configuredPrefillChunkLayers: this.#state.runtimeConfig?.inference?.session?.prefillChunkLayers,
|
|
2975
3270
|
hasGpuSplitPerLayerInputs: context.perLayerInputsSession?.materialization === 'gpu_split_tables',
|
|
2976
3271
|
numTokens,
|
|
2977
3272
|
});
|
|
@@ -3028,6 +3323,7 @@ export class PipelineGenerator {
|
|
|
3028
3323
|
currentRecorder,
|
|
3029
3324
|
'prefill_checkpoint_carry'
|
|
3030
3325
|
);
|
|
3326
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3031
3327
|
await currentRecorder.submitAndWait();
|
|
3032
3328
|
await recordProfile(currentRecorder);
|
|
3033
3329
|
currentRecorder = undefined;
|
|
@@ -3040,6 +3336,7 @@ export class PipelineGenerator {
|
|
|
3040
3336
|
'prefill_trace_layer_health_carry'
|
|
3041
3337
|
);
|
|
3042
3338
|
const traceSubmitStart = performance.now();
|
|
3339
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3043
3340
|
await currentRecorder.submitAndWait();
|
|
3044
3341
|
await recordProfile(currentRecorder);
|
|
3045
3342
|
prefillSubmitWaitMs += performance.now() - traceSubmitStart;
|
|
@@ -3113,6 +3410,7 @@ export class PipelineGenerator {
|
|
|
3113
3410
|
this.#state.modelConfig
|
|
3114
3411
|
);
|
|
3115
3412
|
const chunkSubmitStart = performance.now();
|
|
3413
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3116
3414
|
if (chunkSubmitMode === 'async' && !opts.profile) {
|
|
3117
3415
|
currentRecorder.submit({ cleanup: 'deferred' });
|
|
3118
3416
|
} else {
|
|
@@ -3171,6 +3469,7 @@ export class PipelineGenerator {
|
|
|
3171
3469
|
currentRecorder,
|
|
3172
3470
|
'prefill_tsir_drain_carry'
|
|
3173
3471
|
);
|
|
3472
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3174
3473
|
await currentRecorder.submitAndWait();
|
|
3175
3474
|
await recordProfile(currentRecorder);
|
|
3176
3475
|
currentRecorder = undefined;
|
|
@@ -3181,13 +3480,14 @@ export class PipelineGenerator {
|
|
|
3181
3480
|
if (this.#state.finitenessBuffer) {
|
|
3182
3481
|
if (currentRecorder) {
|
|
3183
3482
|
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3184
|
-
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3483
|
+
currentHiddenBuffer,
|
|
3484
|
+
currentRecorder,
|
|
3485
|
+
'prefill_finiteness_carry'
|
|
3486
|
+
);
|
|
3487
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3488
|
+
await currentRecorder.submitAndWait();
|
|
3489
|
+
await recordProfile(currentRecorder);
|
|
3490
|
+
currentRecorder = undefined;
|
|
3191
3491
|
}
|
|
3192
3492
|
const isInfiniteData = await readBuffer(this.#state.finitenessBuffer, 16);
|
|
3193
3493
|
const u32 = new Uint32Array(isInfiniteData.buffer, isInfiniteData.byteOffset, 4);
|
|
@@ -3251,6 +3551,7 @@ export class PipelineGenerator {
|
|
|
3251
3551
|
let usedRecordedLogits = false;
|
|
3252
3552
|
const lmHead = this.#state.weights.get('lm_head');
|
|
3253
3553
|
const canRecordLogits = !!currentRecorder
|
|
3554
|
+
&& selectedLogitTokenIds === null
|
|
3254
3555
|
&& !!lmHead
|
|
3255
3556
|
&& !isCpuWeightBuffer(lmHead)
|
|
3256
3557
|
&& !this.#state.disableRecordedLogits
|
|
@@ -3268,6 +3569,7 @@ export class PipelineGenerator {
|
|
|
3268
3569
|
logitsVocabSize = recorded.vocabSize;
|
|
3269
3570
|
usedRecordedLogits = true;
|
|
3270
3571
|
|
|
3572
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3271
3573
|
await currentRecorder.submitAndWait();
|
|
3272
3574
|
await recordProfile(currentRecorder);
|
|
3273
3575
|
|
|
@@ -3329,6 +3631,7 @@ export class PipelineGenerator {
|
|
|
3329
3631
|
currentRecorder,
|
|
3330
3632
|
'prefill_logits_carry'
|
|
3331
3633
|
);
|
|
3634
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3332
3635
|
await currentRecorder.submitAndWait();
|
|
3333
3636
|
await recordProfile(currentRecorder);
|
|
3334
3637
|
}
|
|
@@ -3342,13 +3645,17 @@ export class PipelineGenerator {
|
|
|
3342
3645
|
undefined,
|
|
3343
3646
|
debugCheckBuffer,
|
|
3344
3647
|
this.#state.runtimeConfig.shared.debug.probes,
|
|
3345
|
-
{ lastPositionOnly: true },
|
|
3648
|
+
{ lastPositionOnly: true, selectedTokenIds: selectedLogitTokenIds },
|
|
3346
3649
|
this.#state.operatorDiagnostics
|
|
3347
3650
|
);
|
|
3348
3651
|
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3652
|
+
if (selectedLogitTokenIds !== null) {
|
|
3653
|
+
lastLogits = logits;
|
|
3654
|
+
} else {
|
|
3655
|
+
lastLogits = logits.length === logitsVocabSize
|
|
3656
|
+
? logits
|
|
3657
|
+
: extractLastPositionLogits(logits, numTokens, logitsVocabSize);
|
|
3658
|
+
}
|
|
3352
3659
|
releaseBuffer(currentHiddenBuffer);
|
|
3353
3660
|
if (tracePrefillEnabled) {
|
|
3354
3661
|
trace.perf('Prefill logits complete', {
|
|
@@ -3373,11 +3680,11 @@ export class PipelineGenerator {
|
|
|
3373
3680
|
}
|
|
3374
3681
|
}
|
|
3375
3682
|
|
|
3376
|
-
if (opts.debug) {
|
|
3683
|
+
if (opts.debug && selectedLogitTokenIds === null) {
|
|
3377
3684
|
logitsSanity(lastLogits, 'Prefill', (tokens) => resolveTokenText(this.#state.tokenizer, tokens));
|
|
3378
3685
|
}
|
|
3379
3686
|
if (isTraceEnabled('logits')) {
|
|
3380
|
-
trace.logits('PREFILL_LOGITS_HEALTH', getLogitsHealth(lastLogits));
|
|
3687
|
+
trace.logits(selectedLogitTokenIds === null ? 'PREFILL_LOGITS_HEALTH' : 'PREFILL_SELECTED_LOGITS_HEALTH', getLogitsHealth(lastLogits));
|
|
3381
3688
|
}
|
|
3382
3689
|
|
|
3383
3690
|
if (opts.debug) {
|