doppler-gpu 0.4.7 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +93 -144
- package/models/catalog.json +1575 -0
- package/package.json +37 -12
- package/src/cli/doppler-cli.js +28 -7
- package/src/client/doppler-registry.json +2 -2
- package/src/client/runtime/index.js +25 -2
- package/src/client/runtime/model-session.d.ts +31 -1
- package/src/client/runtime/model-session.js +31 -0
- package/src/client/runtime/model-source.d.ts +19 -2
- package/src/client/runtime/model-source.js +45 -5
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
- package/src/config/kernel-path-loader.js +1 -0
- package/src/config/kernels/kernel-ref-digests.js +26 -17
- package/src/config/kernels/registry.json +448 -27
- package/src/config/merge.js +11 -0
- package/src/config/param-validator.js +25 -0
- package/src/config/runtime/default.json +3 -1
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
- package/src/config/runtime/profiles/rerank-stability.json +43 -0
- package/src/config/runtime/profiles/rerank-throughput.json +40 -0
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
- package/src/config/schema/doppler.schema.js +40 -5
- package/src/config/schema/execution-v1.schema.d.ts +41 -0
- package/src/config/schema/execution-v1.schema.js +10 -0
- package/src/config/schema/index.d.ts +3 -1
- package/src/config/schema/speculation-self.schema.d.ts +16 -0
- package/src/config/transforms/capability-transform-resolver.js +7 -1
- package/src/config/transforms/execution-graph-transforms.js +2 -2
- package/src/debug/config.d.ts +4 -0
- package/src/experimental/orchestration/index.d.ts +10 -1
- package/src/experimental/training/suite.js +1 -1
- package/src/formats/rdrr/classification.js +3 -0
- package/src/formats/rdrr/integrity.d.ts +0 -2
- package/src/formats/rdrr/integrity.js +0 -2
- package/src/formats/rdrr/types.d.ts +1 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +75 -16
- package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
- package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
- package/src/gpu/kernels/fused_ffn.d.ts +32 -0
- package/src/gpu/kernels/fused_ffn.js +540 -19
- package/src/gpu/kernels/fused_ffn.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
- package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
- package/src/gpu/kernels/index.d.ts +21 -0
- package/src/gpu/kernels/index.js +13 -0
- package/src/gpu/kernels/linear-attention-core.js +413 -15
- package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
- package/src/gpu/kernels/lm-head-argmax.js +153 -57
- package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
- package/src/gpu/kernels/lm-head-select-logits.js +186 -0
- package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
- package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
- package/src/gpu/kernels/matmul-selection.js +5 -1
- package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
- package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
- package/src/gpu/kernels/rmsnorm-pair.js +113 -0
- package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
- package/src/gpu/kernels/rmsnorm-stats.js +250 -0
- package/src/gpu/kernels/silu.wgsl +7 -1
- package/src/gpu/kernels/silu_f16.wgsl +7 -1
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
- package/src/gpu/tensor.d.ts +1 -0
- package/src/index.d.ts +1 -1
- package/src/index.js +1 -1
- package/src/inference/browser-harness-suite-helpers.js +1 -1
- package/src/inference/browser-harness-text-helpers.js +251 -20
- package/src/inference/browser-harness.js +221 -3
- package/src/inference/kv-cache/base.js +42 -1
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
- package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
- package/src/inference/pipelines/text/attention/projections.js +174 -4
- package/src/inference/pipelines/text/attention/record.js +44 -21
- package/src/inference/pipelines/text/attention/run.js +18 -3
- package/src/inference/pipelines/text/config.d.ts +18 -0
- package/src/inference/pipelines/text/config.js +100 -0
- package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
- package/src/inference/pipelines/text/execution-plan.js +37 -0
- package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
- package/src/inference/pipelines/text/execution-v1.js +122 -8
- package/src/inference/pipelines/text/ffn/dense.js +239 -21
- package/src/inference/pipelines/text/ffn/standard.js +121 -38
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
- package/src/inference/pipelines/text/generator-helpers.js +2 -0
- package/src/inference/pipelines/text/generator-runtime.js +1 -0
- package/src/inference/pipelines/text/generator-steps.js +2 -2
- package/src/inference/pipelines/text/generator.d.ts +18 -1
- package/src/inference/pipelines/text/generator.js +297 -19
- package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
- package/src/inference/pipelines/text/init.d.ts +22 -3
- package/src/inference/pipelines/text/init.js +2 -2
- package/src/inference/pipelines/text/layer.js +87 -8
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
- package/src/inference/pipelines/text/linear-attention.js +173 -16
- package/src/inference/pipelines/text/logits/gpu.js +2 -2
- package/src/inference/pipelines/text/logits/index.js +62 -1
- package/src/inference/pipelines/text/model-load.js +6 -0
- package/src/inference/pipelines/text/ops.js +22 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
- package/src/inference/pipelines/text/probes.js +9 -4
- package/src/inference/pipelines/text/state.d.ts +4 -4
- package/src/inference/pipelines/text/state.js +4 -0
- package/src/inference/pipelines/text/types.d.ts +27 -2
- package/src/inference/pipelines/text.d.ts +21 -1
- package/src/inference/pipelines/text.js +40 -14
- package/src/inference/tokenizers/bundled.js +45 -23
- package/src/inference/tokenizers/types.d.ts +7 -4
- package/src/loader/doppler-loader.js +25 -2
- package/src/loader/index.d.ts +3 -1
- package/src/loader/index.js +2 -0
- package/src/loader/loader-types.d.ts +3 -1
- package/src/loader/manifest-config.d.ts +18 -0
- package/src/loader/manifest-config.js +11 -3
- package/src/loader/shard-cache.d.ts +2 -0
- package/src/loader/shard-cache.js +29 -1
- package/src/models/gemma4.js +8 -0
- package/src/rules/inference/capability-transforms.rules.json +145 -20
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/fused-ffn.rules.json +1 -1
- package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/rules/kernels/silu.rules.json +1 -0
- package/src/rules/rule-registry.js +2 -0
- package/src/storage/artifact-storage-context.js +36 -0
- package/src/tooling/node-browser-command-runner.js +42 -7
- package/src/tooling/opfs-cache.d.ts +21 -1
- package/src/tooling/opfs-cache.js +168 -32
- package/src/tooling-exports/storage.d.ts +1 -1
- package/src/tooling-exports/storage.js +1 -1
- package/src/types/gpu.d.ts +25 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +1593 -0
- package/tests/kernels/browser/test-page.js +2629 -0
- package/tests/kernels/harness/benchmark.js +225 -0
- package/tests/kernels/harness/buffer-utils.js +88 -0
- package/tests/kernels/harness/tolerance.js +174 -0
- package/tests/kernels/reference/adam.js +15 -0
- package/tests/kernels/reference/attention.js +158 -0
- package/tests/kernels/reference/backward.js +96 -0
- package/tests/kernels/reference/bias-add.js +20 -0
- package/tests/kernels/reference/conv2d.js +58 -0
- package/tests/kernels/reference/cross-entropy.js +26 -0
- package/tests/kernels/reference/dequant.js +312 -0
- package/tests/kernels/reference/embed.js +15 -0
- package/tests/kernels/reference/gather.js +61 -0
- package/tests/kernels/reference/gelu.js +37 -0
- package/tests/kernels/reference/groupnorm.js +36 -0
- package/tests/kernels/reference/index.js +89 -0
- package/tests/kernels/reference/layernorm.js +22 -0
- package/tests/kernels/reference/matmul.js +57 -0
- package/tests/kernels/reference/modulate.js +24 -0
- package/tests/kernels/reference/moe-gather.js +83 -0
- package/tests/kernels/reference/pixel-shuffle.js +20 -0
- package/tests/kernels/reference/residual.js +55 -0
- package/tests/kernels/reference/rmsnorm.js +52 -0
- package/tests/kernels/reference/rope.js +79 -0
- package/tests/kernels/reference/sample.js +97 -0
- package/tests/kernels/reference/scatter-add.js +58 -0
- package/tests/kernels/reference/silu.js +52 -0
- package/tests/kernels/reference/softmax.js +86 -0
- package/tests/kernels/reference/split-qkv.js +58 -0
- package/tests/kernels/reference/topk.js +88 -0
- package/tests/kernels/reference/upsample2d.js +37 -0
- package/BRANDING.md +0 -14
- package/SECURITY.md +0 -19
- package/assets/doppler-runtime-map.svg +0 -82
- package/src/bootstrap.js +0 -30
- package/src/cli/doppler-cli.d.ts +0 -46
- package/src/cli/doppler-quickstart.d.ts +0 -45
- package/src/cli/doppler-serve.d.ts +0 -210
- package/src/cli/runtime-profiles.d.ts +0 -40
- package/src/client/doppler-api.browser.d.ts +0 -1
- package/src/client/doppler-api.browser.js +0 -25
- package/src/client/doppler-registry.d.ts +0 -54
- package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
- package/src/client/runtime/source-runtime.d.ts +0 -26
- package/src/client/runtime/storage.d.ts +0 -5
- package/src/config/README.md +0 -61
- package/src/config/conversion/README.md +0 -62
- package/src/config/goal-completion-matrix.json +0 -552
- package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
- package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
- package/src/config/kernels/moe/mixtral.paths.json +0 -46
- package/src/config/kernels/registry.d.ts +0 -51
- package/src/config/merge-contract-check.d.ts +0 -16
- package/src/config/merge-contract-check.js +0 -319
- package/src/config/merge-helpers.d.ts +0 -60
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
- package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
- package/src/config/param-categories.d.ts +0 -8
- package/src/config/param-validator.d.ts +0 -33
- package/src/config/platforms/loader.d.ts +0 -38
- package/src/config/quantization-contract-check.d.ts +0 -12
- package/src/config/quantization-contract-check.js +0 -91
- package/src/config/runtime/README.md +0 -28
- package/src/config/runtime-merge.d.ts +0 -5
- package/src/config/schema/claim-evidence-contract.schema.json +0 -93
- package/src/config/schema/command-surface-contract.schema.json +0 -81
- package/src/config/schema/goal-completion-matrix.schema.json +0 -223
- package/src/config/schema/model-artifact-contract.schema.json +0 -119
- package/src/config/schema/platform.schema.js +0 -1
- package/src/config/schema/policy-schema-registry.json +0 -31
- package/src/config/schema/subsystem-support-contract.schema.json +0 -48
- package/src/config/support-tiers/subsystems.json +0 -502
- package/src/config/supported-operations.d.ts +0 -14
- package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
- package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
- package/src/converter/conversion-plan.d.ts +0 -56
- package/src/converter/index.d.ts +0 -98
- package/src/converter/index.js +0 -59
- package/src/converter/parsers/diffusion.d.ts +0 -50
- package/src/converter/parsers/gguf.d.ts +0 -31
- package/src/converter/parsers/index.d.ts +0 -21
- package/src/converter/parsers/index.js +0 -12
- package/src/converter/parsers/transformer.d.ts +0 -31
- package/src/converter/quantization-info.d.ts +0 -38
- package/src/converter/rope-config.d.ts +0 -16
- package/src/converter/tokenizer-utils.d.ts +0 -12
- package/src/debug/reference/README.md +0 -114
- package/src/debug/reference/hf_attn_debug.py +0 -114
- package/src/debug/reference/hf_embed_check.py +0 -89
- package/src/debug/reference/hf_layer_out.py +0 -100
- package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
- package/src/debug/reference/hf_rope_check.py +0 -116
- package/src/debug/reference/hf_weights.py +0 -75
- package/src/errors/doppler-error.d.ts +0 -20
- package/src/experimental/bridge/extension/background.d.ts +0 -14
- package/src/experimental/bridge/extension/background.js +0 -168
- package/src/experimental/bridge/extension/manifest.json +0 -34
- package/src/experimental/bridge/extension-client.d.ts +0 -114
- package/src/experimental/bridge/index.d.ts +0 -69
- package/src/experimental/bridge/node-bridge.js +0 -24
- package/src/experimental/bridge/protocol.d.ts +0 -96
- package/src/experimental/browser/gguf-importer.d.ts +0 -136
- package/src/experimental/browser/gguf-importer.js +0 -532
- package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
- package/src/experimental/browser/quantization.d.ts +0 -74
- package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
- package/src/experimental/browser/shard-io-browser.d.ts +0 -57
- package/src/experimental/browser/shard-io-browser.js +0 -89
- package/src/experimental/browser/tensor-source-download.d.ts +0 -27
- package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
- package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
- package/src/experimental/distribution/p2p-transport.js +0 -75
- package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
- package/src/experimental/hotswap/manifest.d.ts +0 -42
- package/src/experimental/hotswap/model-swap.js +0 -42
- package/src/experimental/hotswap/runtime.d.ts +0 -31
- package/src/experimental/training/README.md +0 -164
- package/src/experimental/training/artifacts.d.ts +0 -160
- package/src/experimental/training/attention-backward.d.ts +0 -30
- package/src/experimental/training/attention-forward.d.ts +0 -22
- package/src/experimental/training/attention-forward.js +0 -82
- package/src/experimental/training/autograd.d.ts +0 -51
- package/src/experimental/training/backward-kernels.js +0 -39
- package/src/experimental/training/checkpoint-watch.d.ts +0 -8
- package/src/experimental/training/checkpoint.d.ts +0 -36
- package/src/experimental/training/clip.d.ts +0 -9
- package/src/experimental/training/dataloader.d.ts +0 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
- package/src/experimental/training/datasets/gepa-frontier.js +0 -220
- package/src/experimental/training/datasets/index.d.ts +0 -24
- package/src/experimental/training/datasets/index.js +0 -13
- package/src/experimental/training/datasets/jsonl.d.ts +0 -11
- package/src/experimental/training/datasets/reploid.d.ts +0 -3
- package/src/experimental/training/datasets/reploid.js +0 -36
- package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
- package/src/experimental/training/datasets/teacher-traces.js +0 -279
- package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
- package/src/experimental/training/datasets/token-batch.d.ts +0 -21
- package/src/experimental/training/datasets/token-batch.js +0 -52
- package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
- package/src/experimental/training/datasets/translation-pairs.js +0 -49
- package/src/experimental/training/distillation/artifacts.d.ts +0 -71
- package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
- package/src/experimental/training/distillation/dataset.d.ts +0 -59
- package/src/experimental/training/distillation/eval.d.ts +0 -34
- package/src/experimental/training/distillation/index.d.ts +0 -29
- package/src/experimental/training/distillation/runtime.d.ts +0 -20
- package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
- package/src/experimental/training/distillation/stage-a.d.ts +0 -45
- package/src/experimental/training/distillation/stage-b.d.ts +0 -24
- package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
- package/src/experimental/training/distillation/suite-data.d.ts +0 -45
- package/src/experimental/training/export.d.ts +0 -41
- package/src/experimental/training/index.d.ts +0 -74
- package/src/experimental/training/index.js +0 -63
- package/src/experimental/training/lora-pipeline.d.ts +0 -183
- package/src/experimental/training/lora.d.ts +0 -19
- package/src/experimental/training/loss-scaling.d.ts +0 -21
- package/src/experimental/training/loss.d.ts +0 -10
- package/src/experimental/training/objectives/base.d.ts +0 -58
- package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
- package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
- package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
- package/src/experimental/training/objectives/index.d.ts +0 -12
- package/src/experimental/training/objectives/index.js +0 -6
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
- package/src/experimental/training/operator-agent-eval.d.ts +0 -65
- package/src/experimental/training/operator-artifacts.d.ts +0 -62
- package/src/experimental/training/operator-command.d.ts +0 -5
- package/src/experimental/training/operator-eval.d.ts +0 -48
- package/src/experimental/training/operator-scoreboard.d.ts +0 -5
- package/src/experimental/training/optimizer.d.ts +0 -22
- package/src/experimental/training/runner.d.ts +0 -249
- package/src/experimental/training/suite.d.ts +0 -299
- package/src/experimental/training/tensor-factory.d.ts +0 -9
- package/src/experimental/training/trainer.d.ts +0 -89
- package/src/experimental/training/ul_dataset.d.ts +0 -47
- package/src/experimental/training/ul_schedule.d.ts +0 -6
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
- package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
- package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
- package/src/experimental/training/workload-packs/registry.json +0 -278
- package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
- package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
- package/src/experimental/training/workloads.d.ts +0 -201
- package/src/formats/gguf/types.d.ts +0 -74
- package/src/formats/index.d.ts +0 -75
- package/src/formats/index.js +0 -19
- package/src/formats/litert/types.d.ts +0 -74
- package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
- package/src/formats/rdrr/distributed/validation.d.ts +0 -23
- package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
- package/src/formats/safetensors/types.d.ts +0 -67
- package/src/formats/tflite/types.d.ts +0 -61
- package/src/formats/tokenizer/index.d.ts +0 -5
- package/src/formats/tokenizer/index.js +0 -3
- package/src/formats/tokenizer/types.d.ts +0 -9
- package/src/gpu/kernel-runtime.d.ts +0 -13
- package/src/gpu/kernel-selection-cache.d.ts +0 -13
- package/src/gpu/kernel-selection-log.d.ts +0 -4
- package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
- package/src/gpu/kernel-tuner/cache.d.ts +0 -55
- package/src/gpu/kernel-tuner/index.d.ts +0 -59
- package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
- package/src/gpu/kernel-tuner/types.d.ts +0 -101
- package/src/gpu/kernel-tuner/types.js +0 -4
- package/src/gpu/kernel-tuner.d.ts +0 -33
- package/src/gpu/kernels/README.md +0 -166
- package/src/gpu/kernels/check-finiteness.d.ts +0 -15
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
- package/src/gpu/kernels/check-stop.d.ts +0 -31
- package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
- package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
- package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
- package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
- package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
- package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
- package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
- package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
- package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
- package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
- package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
- package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
- package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
- package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
- package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
- package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
- package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
- package/src/gpu/kernels/constants.d.ts +0 -129
- package/src/gpu/kernels/dispatch.d.ts +0 -72
- package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
- package/src/gpu/kernels/kernel-base.d.ts +0 -33
- package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
- package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
- package/src/gpu/kernels/matmul-selection.d.ts +0 -87
- package/src/gpu/kernels/rep-penalty.d.ts +0 -21
- package/src/gpu/kernels/rule-matcher.d.ts +0 -30
- package/src/gpu/kernels/rule-registry.d.ts +0 -7
- package/src/gpu/kernels/types.js +0 -4
- package/src/gpu/perf-guards.d.ts +0 -13
- package/src/gpu/profiler.d.ts +0 -93
- package/src/gpu/readback-utils.d.ts +0 -16
- package/src/gpu/readback-utils.js +0 -49
- package/src/gpu/submit-probe.d.ts +0 -12
- package/src/index-browser.d.ts +0 -2
- package/src/index-browser.js +0 -2
- package/src/inference/README.md +0 -593
- package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
- package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
- package/src/inference/browser-harness-model-helpers.d.ts +0 -10
- package/src/inference/browser-harness-report-helpers.d.ts +0 -7
- package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
- package/src/inference/browser-harness-text-helpers.d.ts +0 -152
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
- package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
- package/src/inference/network-evolution.js +0 -79
- package/src/inference/pipelines/context.d.ts +0 -21
- package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
- package/src/inference/pipelines/diffusion/index.d.ts +0 -3
- package/src/inference/pipelines/diffusion/index.js +0 -3
- package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
- package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
- package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
- package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
- package/src/inference/pipelines/diffusion/types.js +0 -1
- package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
- package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
- package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
- package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
- package/src/inference/pipelines/dream/pipeline.js +0 -8
- package/src/inference/pipelines/energy/index.d.ts +0 -1
- package/src/inference/pipelines/energy/index.js +0 -1
- package/src/inference/pipelines/energy/types.js +0 -1
- package/src/inference/pipelines/energy-head/index.d.ts +0 -6
- package/src/inference/pipelines/energy-head/index.js +0 -6
- package/src/inference/pipelines/factory.d.ts +0 -10
- package/src/inference/pipelines/index.d.ts +0 -22
- package/src/inference/pipelines/index.js +0 -19
- package/src/inference/pipelines/registry.d.ts +0 -15
- package/src/inference/pipelines/rng.d.ts +0 -2
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
- package/src/inference/pipelines/text/attention/index.d.ts +0 -24
- package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
- package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
- package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
- package/src/inference/pipelines/text/attention/record.d.ts +0 -36
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
- package/src/inference/pipelines/text/attention/run.d.ts +0 -38
- package/src/inference/pipelines/text/attention/types.d.ts +0 -125
- package/src/inference/pipelines/text/attention.d.ts +0 -23
- package/src/inference/pipelines/text/attention.js +0 -12
- package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
- package/src/inference/pipelines/text/buffer-types.js +0 -4
- package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
- package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
- package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
- package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
- package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
- package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
- package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
- package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
- package/src/inference/pipelines/text/embed.d.ts +0 -69
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
- package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
- package/src/inference/pipelines/text/failure-reduction.js +0 -180
- package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
- package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
- package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
- package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
- package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
- package/src/inference/pipelines/text/ffn.d.ts +0 -31
- package/src/inference/pipelines/text/ffn.js +0 -18
- package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
- package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
- package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
- package/src/inference/pipelines/text/index.d.ts +0 -5
- package/src/inference/pipelines/text/index.js +0 -6
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
- package/src/inference/pipelines/text/layer.d.ts +0 -118
- package/src/inference/pipelines/text/load-errors.d.ts +0 -21
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
- package/src/inference/pipelines/text/logits/types.js +0 -4
- package/src/inference/pipelines/text/logits.d.ts +0 -27
- package/src/inference/pipelines/text/logits.js +0 -16
- package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
- package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
- package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
- package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
- package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
- package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
- package/src/inference/pipelines/text/operator-events.d.ts +0 -125
- package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
- package/src/inference/pipelines/text/ops.d.ts +0 -222
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
- package/src/inference/pipelines/text/probes.d.ts +0 -28
- package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
- package/src/inference/pipelines/text/stage-names.d.ts +0 -88
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
- package/src/inference/pipelines/text/types.js +0 -4
- package/src/inference/tokenizers/base.d.ts +0 -44
- package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
- package/src/inference/tokenizers/bpe.d.ts +0 -27
- package/src/inference/tokenizers/bundled.d.ts +0 -67
- package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
- package/src/inference/tokenizers/types.js +0 -7
- package/src/loader/embedding-limit-preflight.d.ts +0 -31
- package/src/loader/per-layer-input-loader.d.ts +0 -30
- package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
- package/src/loader/tensors/source-transform.d.ts +0 -65
- package/src/memory/address-table.d.ts +0 -96
- package/src/rules/execution-rules-contract-check.d.ts +0 -21
- package/src/rules/layer-pattern-contract-check.d.ts +0 -17
- package/src/rules/rule-registry.d.ts +0 -77
- package/src/storage/artifact-storage-context.d.ts +0 -63
- package/src/storage/backends/idb-store.d.ts +0 -52
- package/src/storage/backends/memory-store.d.ts +0 -36
- package/src/storage/backends/opfs-store.d.ts +0 -41
- package/src/storage/blake3.d.ts +0 -17
- package/src/storage/download-types.d.ts +0 -157
- package/src/storage/downloader.d.ts +0 -103
- package/src/storage/index.d.ts +0 -274
- package/src/storage/index.js +0 -201
- package/src/storage/preflight.d.ts +0 -144
- package/src/storage/preflight.js +0 -360
- package/src/storage/quickstart-downloader.d.ts +0 -157
- package/src/storage/quickstart-downloader.js +0 -289
- package/src/storage/source-artifact-store.d.ts +0 -59
- package/src/sw.js +0 -184
- package/src/tooling/command-api-constants.d.ts +0 -9
- package/src/tooling/command-api-family-normalizers.d.ts +0 -10
- package/src/tooling/command-api-helpers.d.ts +0 -23
- package/src/tooling/command-runner-shared.d.ts +0 -89
- package/src/tooling/conversion-config-materializer.d.ts +0 -14
- package/src/tooling/conversion-config-materializer.js +0 -96
- package/src/tooling/diagnose-runner.d.ts +0 -28
- package/src/tooling/hf-registry-utils.d.ts +0 -55
- package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
- package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
- package/src/tooling/kernel-path-builder/index.d.ts +0 -65
- package/src/tooling/kernel-path-builder/index.js +0 -1361
- package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
- package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
- package/src/tooling/lean-execution-contract-runner.js +0 -158
- package/src/tooling/lean-execution-contract.d.ts +0 -16
- package/src/tooling/lean-execution-contract.js +0 -81
- package/src/tooling/litert-package-runtime.d.ts +0 -46
- package/src/tooling/node-convert-worker-pool.d.ts +0 -16
- package/src/tooling/node-convert-worker.d.ts +0 -1
- package/src/tooling/node-converter.d.ts +0 -1
- package/src/tooling/node-file-fetch.d.ts +0 -1
- package/src/tooling/node-source-runtime.d.ts +0 -24
- package/src/tooling/node-webgpu.d.ts +0 -17
- package/src/tooling/precision-replay-math.js +0 -174
- package/src/tooling/program-bundle-host.d.ts +0 -16
- package/src/tooling/program-bundle-host.js +0 -20
- package/src/tooling/runtime-input-composition.d.ts +0 -37
- package/src/tooling/source-package-profiles.d.ts +0 -50
- package/src/tooling/source-runtime-converter-config.d.ts +0 -19
- package/src/tooling/source-runtime-materializer.d.ts +0 -6
- package/src/tooling/source-runtime-materializer.js +0 -89
- package/src/tooling-exports.browser.d.ts +0 -7
- package/src/types/chrome.d.ts +0 -36
- package/src/types/chrome.js +0 -1
- package/src/types/index.d.ts +0 -1
- package/src/types/index.js +0 -1
- package/src/utils/array-buffer.d.ts +0 -4
- package/src/utils/canonical-hash.d.ts +0 -7
- package/src/utils/clone-json.d.ts +0 -1
- package/src/utils/hf-resolve-url.d.ts +0 -16
- package/src/utils/index.d.ts +0 -10
- package/src/utils/index.js +0 -10
- package/src/utils/load-json.d.ts +0 -5
- package/src/utils/plain-object.d.ts +0 -1
- package/src/utils/sha256.d.ts +0 -5
- package/src/utils/stable-sort-object.d.ts +0 -1
|
@@ -25,12 +25,26 @@ import { embed, isRangeBackedCpuEmbeddingSource, normalizeRangeBytes, decodeRang
|
|
|
25
25
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
26
26
|
import { getDevice } from '../../../gpu/device.js';
|
|
27
27
|
import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
28
|
+
import { f16ToF32 } from '../../../loader/dtype-utils.js';
|
|
28
29
|
|
|
29
30
|
const pleRuntimeCache = new WeakMap();
|
|
30
31
|
const pleRangeRowCache = new WeakMap();
|
|
32
|
+
let f16ToF32Lookup = null;
|
|
31
33
|
const PLE_ACTIVATION_STATIC_QMIN = -128;
|
|
32
34
|
const PLE_ACTIVATION_STATIC_QMAX = 127;
|
|
33
35
|
|
|
36
|
+
function getF16ToF32Lookup() {
|
|
37
|
+
if (f16ToF32Lookup) {
|
|
38
|
+
return f16ToF32Lookup;
|
|
39
|
+
}
|
|
40
|
+
const lookup = new Float32Array(1 << 16);
|
|
41
|
+
for (let value = 0; value < lookup.length; value += 1) {
|
|
42
|
+
lookup[value] = f16ToF32(value);
|
|
43
|
+
}
|
|
44
|
+
f16ToF32Lookup = lookup;
|
|
45
|
+
return lookup;
|
|
46
|
+
}
|
|
47
|
+
|
|
34
48
|
function getPerLayerInputWeights(context) {
|
|
35
49
|
const weights = context.weights.get('per_layer_inputs');
|
|
36
50
|
if (!weights || typeof weights !== 'object') {
|
|
@@ -625,6 +639,28 @@ function getPleHotCachePolicy(sessionConfig) {
|
|
|
625
639
|
return { mode: 'prepared_tokens', maxTokens, maxBytes, outputDtype };
|
|
626
640
|
}
|
|
627
641
|
|
|
642
|
+
export function resolvePleHotVocabularyCapacity({
|
|
643
|
+
maxTokens,
|
|
644
|
+
maxBytes,
|
|
645
|
+
numLayers,
|
|
646
|
+
hiddenSize,
|
|
647
|
+
bytesPerElement,
|
|
648
|
+
vocabSize,
|
|
649
|
+
}) {
|
|
650
|
+
const bytesPerHotRow = numLayers * hiddenSize * bytesPerElement;
|
|
651
|
+
const tokenIndexMapBytes = vocabSize * Uint32Array.BYTES_PER_ELEMENT;
|
|
652
|
+
const tableBudgetBytes = Math.max(0, maxBytes - tokenIndexMapBytes);
|
|
653
|
+
const maxTableRows = bytesPerHotRow > 0
|
|
654
|
+
? Math.floor(tableBudgetBytes / bytesPerHotRow)
|
|
655
|
+
: 0;
|
|
656
|
+
const maxHotTokens = Math.max(0, Math.min(maxTokens, maxTableRows - 1));
|
|
657
|
+
return {
|
|
658
|
+
maxHotTokens,
|
|
659
|
+
bytesPerHotRow,
|
|
660
|
+
tokenIndexMapBytes,
|
|
661
|
+
};
|
|
662
|
+
}
|
|
663
|
+
|
|
628
664
|
export function getPleHotVocabularyRuntime(context) {
|
|
629
665
|
const perLayerInputWeights = context?.weights?.get?.('per_layer_inputs');
|
|
630
666
|
if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
|
|
@@ -1095,19 +1131,32 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
|
1095
1131
|
return null;
|
|
1096
1132
|
}
|
|
1097
1133
|
|
|
1134
|
+
const bytesPerElement = policy.outputDtype === 'f16' ? 2 : 4;
|
|
1135
|
+
const capacity = resolvePleHotVocabularyCapacity({
|
|
1136
|
+
maxTokens: policy.maxTokens,
|
|
1137
|
+
maxBytes: policy.maxBytes,
|
|
1138
|
+
numLayers,
|
|
1139
|
+
hiddenSize: hiddenSizePerLayerInput,
|
|
1140
|
+
bytesPerElement,
|
|
1141
|
+
vocabSize: vocabSizePerLayerInput,
|
|
1142
|
+
});
|
|
1143
|
+
if (capacity.maxHotTokens <= 0) {
|
|
1144
|
+
return null;
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1098
1147
|
const tokenizer = context?.tokenizer ?? null;
|
|
1099
1148
|
const tokenizerHotTokenIds = typeof tokenizer?.getHotTokenIds === 'function'
|
|
1100
|
-
? tokenizer.getHotTokenIds(
|
|
1149
|
+
? tokenizer.getHotTokenIds(capacity.maxHotTokens)
|
|
1101
1150
|
: null;
|
|
1102
1151
|
const seedTokenIds = resolvePleHotVocabularySeedTokenIds(
|
|
1103
1152
|
context,
|
|
1104
|
-
|
|
1153
|
+
capacity.maxHotTokens,
|
|
1105
1154
|
vocabSizePerLayerInput
|
|
1106
1155
|
);
|
|
1107
1156
|
const hotTokenIds = mergePleHotVocabularyTokenIds(
|
|
1108
1157
|
seedTokenIds,
|
|
1109
1158
|
tokenizerHotTokenIds,
|
|
1110
|
-
|
|
1159
|
+
capacity.maxHotTokens,
|
|
1111
1160
|
vocabSizePerLayerInput
|
|
1112
1161
|
);
|
|
1113
1162
|
if (hotTokenIds.length === 0) {
|
|
@@ -1119,7 +1168,8 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
|
1119
1168
|
const cached = perLayerInputWeights.embedTokensPerLayerHotRuntime ?? null;
|
|
1120
1169
|
if (
|
|
1121
1170
|
cached
|
|
1122
|
-
&& cached.maxTokens ===
|
|
1171
|
+
&& cached.maxTokens === capacity.maxHotTokens
|
|
1172
|
+
&& cached.maxBytes === policy.maxBytes
|
|
1123
1173
|
&& cached.outputDtype === policy.outputDtype
|
|
1124
1174
|
&& cached.vocabSize === vocabSizePerLayerInput
|
|
1125
1175
|
&& cached.numLayers === numLayers
|
|
@@ -1136,13 +1186,13 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
|
1136
1186
|
|
|
1137
1187
|
const sourceDtype = requirePleSourceDtype(
|
|
1138
1188
|
embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype,
|
|
1139
|
-
'Gemma 4 hot vocabulary cache'
|
|
1140
|
-
[policy.outputDtype]
|
|
1189
|
+
'Gemma 4 hot vocabulary cache'
|
|
1141
1190
|
);
|
|
1142
|
-
|
|
1191
|
+
const expandsF16ToF32 = sourceDtype === 'f16' && policy.outputDtype === 'f32';
|
|
1192
|
+
if (sourceDtype !== policy.outputDtype && !expandsF16ToF32) {
|
|
1143
1193
|
throw new Error(
|
|
1144
|
-
`Gemma 4 hot vocabulary cache
|
|
1145
|
-
`
|
|
1194
|
+
`Gemma 4 hot vocabulary cache cannot convert source dtype "${sourceDtype}" ` +
|
|
1195
|
+
`to output dtype "${policy.outputDtype}".`
|
|
1146
1196
|
);
|
|
1147
1197
|
}
|
|
1148
1198
|
|
|
@@ -1151,7 +1201,6 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
|
1151
1201
|
throw new Error('No GPU device available for Gemma 4 hot vocabulary cache.');
|
|
1152
1202
|
}
|
|
1153
1203
|
|
|
1154
|
-
const bytesPerElement = policy.outputDtype === 'f16' ? 2 : 4;
|
|
1155
1204
|
const totalPerLayerHiddenSize = numLayers * hiddenSizePerLayerInput;
|
|
1156
1205
|
const sentinelIndex = hotTokenIds.length;
|
|
1157
1206
|
const hotRowCount = sentinelIndex + 1;
|
|
@@ -1193,6 +1242,10 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
|
1193
1242
|
}
|
|
1194
1243
|
|
|
1195
1244
|
const { sourceRowBytes } = getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize);
|
|
1245
|
+
const expandedF32Row = expandsF16ToF32
|
|
1246
|
+
? new Float32Array(totalPerLayerHiddenSize)
|
|
1247
|
+
: null;
|
|
1248
|
+
const f16Lookup = expandsF16ToF32 ? getF16ToF32Lookup() : null;
|
|
1196
1249
|
for (let hotIndex = 0; hotIndex < hotTokenIds.length; hotIndex += 1) {
|
|
1197
1250
|
const tokenId = hotTokenIds[hotIndex];
|
|
1198
1251
|
const chunk = normalizeRangeBytes(
|
|
@@ -1211,6 +1264,21 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
|
1211
1264
|
hiddenSizePerLayerInput * bytesPerElement
|
|
1212
1265
|
);
|
|
1213
1266
|
}
|
|
1267
|
+
} else if (sourceDtype === 'f16') {
|
|
1268
|
+
const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
|
|
1269
|
+
for (let valueIdx = 0; valueIdx < totalPerLayerHiddenSize; valueIdx += 1) {
|
|
1270
|
+
expandedF32Row[valueIdx] = f16Lookup[sourceWords[valueIdx]];
|
|
1271
|
+
}
|
|
1272
|
+
for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
|
|
1273
|
+
const sourceStart = layerIdx * hiddenSizePerLayerInput;
|
|
1274
|
+
device.queue.writeBuffer(
|
|
1275
|
+
splitTables[layerIdx].buffer,
|
|
1276
|
+
hotIndex * hiddenSizePerLayerInput * bytesPerElement,
|
|
1277
|
+
expandedF32Row.buffer,
|
|
1278
|
+
sourceStart * bytesPerElement,
|
|
1279
|
+
hiddenSizePerLayerInput * bytesPerElement
|
|
1280
|
+
);
|
|
1281
|
+
}
|
|
1214
1282
|
} else {
|
|
1215
1283
|
const sourceValues = new Float32Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
|
|
1216
1284
|
for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
|
|
@@ -1235,7 +1303,8 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
|
1235
1303
|
|
|
1236
1304
|
const runtime = {
|
|
1237
1305
|
mode: 'tokenizer_scores',
|
|
1238
|
-
maxTokens:
|
|
1306
|
+
maxTokens: capacity.maxHotTokens,
|
|
1307
|
+
maxBytes: policy.maxBytes,
|
|
1239
1308
|
outputDtype: policy.outputDtype,
|
|
1240
1309
|
vocabSize: vocabSizePerLayerInput,
|
|
1241
1310
|
numLayers,
|
|
@@ -164,8 +164,8 @@ async function readTokenRow(buffer, tokenIdx, hiddenSize, dtype, isCpuBuffer) {
|
|
|
164
164
|
return values;
|
|
165
165
|
}
|
|
166
166
|
|
|
167
|
-
function formatProbeStats(stats) {
|
|
168
|
-
|
|
167
|
+
function formatProbeStats(stats, firstNonFiniteIndex = -1) {
|
|
168
|
+
const fields = [
|
|
169
169
|
`min=${formatProbeNumber(stats.min)}`,
|
|
170
170
|
`max=${formatProbeNumber(stats.max)}`,
|
|
171
171
|
`mean=${formatProbeNumber(stats.mean)}`,
|
|
@@ -175,7 +175,11 @@ function formatProbeStats(stats) {
|
|
|
175
175
|
`nan=${stats.nanCount}`,
|
|
176
176
|
`inf=${stats.infCount}`,
|
|
177
177
|
`zero=${stats.zeroCount}`,
|
|
178
|
-
]
|
|
178
|
+
];
|
|
179
|
+
if (firstNonFiniteIndex >= 0) {
|
|
180
|
+
fields.push(`firstNonFinite=${firstNonFiniteIndex}`);
|
|
181
|
+
}
|
|
182
|
+
return fields.join(', ');
|
|
179
183
|
}
|
|
180
184
|
|
|
181
185
|
|
|
@@ -267,7 +271,8 @@ export async function runProbes(stage, buffer, options) {
|
|
|
267
271
|
let statsText = null;
|
|
268
272
|
if (includeStats) {
|
|
269
273
|
const row = await readTokenRow(buffer, tokenIdx, hiddenSize, dtype, isCpuBuffer);
|
|
270
|
-
|
|
274
|
+
const firstNonFiniteIndex = row.findIndex((value) => !Number.isFinite(value));
|
|
275
|
+
statsText = formatProbeStats(computeArrayStats(row), firstNonFiniteIndex);
|
|
271
276
|
}
|
|
272
277
|
|
|
273
278
|
const values = [];
|
|
@@ -28,14 +28,13 @@ import type { DopplerLoader } from '../../../loader/doppler-loader.js';
|
|
|
28
28
|
import type { LoadedEmbeddingPostprocessor } from '../../../loader/final-weights-loader.js';
|
|
29
29
|
import type { CompiledLayerPipeline } from './layer-plan.js';
|
|
30
30
|
import type { ExecutionPlanState } from './execution-plan.js';
|
|
31
|
-
import type {
|
|
31
|
+
import type { ExecutionV1CompiledState } from './execution-v1.js';
|
|
32
32
|
import type { LoRAAdapter } from './lora.js';
|
|
33
33
|
import type { RuntimeConfigSchema, KernelPathSchema } from '../../../config/schema/index.js';
|
|
34
34
|
import type { WeightDebugFlags } from './weights.js';
|
|
35
35
|
import type { LogitsDebugFlags } from './logits/index.js';
|
|
36
36
|
import type { KernelPathSource } from '../../../config/kernel-path-loader.js';
|
|
37
|
-
import type { EmulationContext } from '
|
|
38
|
-
import type { PipelineStorageContext } from './init.js';
|
|
37
|
+
import type { EmulationContext, PipelineStorageContext } from './init.js';
|
|
39
38
|
import type { LinearAttentionRuntime } from './linear-attention.js';
|
|
40
39
|
import type { PerLayerInputWeights } from '../../../loader/loader-types.js';
|
|
41
40
|
|
|
@@ -60,7 +59,7 @@ export class PipelineState {
|
|
|
60
59
|
resolvedKernelPath: KernelPathSchema | null;
|
|
61
60
|
kernelPathSource: KernelPathSource;
|
|
62
61
|
executionPlanState: ExecutionPlanState | null;
|
|
63
|
-
executionV1State:
|
|
62
|
+
executionV1State: ExecutionV1CompiledState | null;
|
|
64
63
|
disableRecordedLogits: boolean;
|
|
65
64
|
disableFusedDecode: boolean;
|
|
66
65
|
|
|
@@ -79,6 +78,7 @@ export class PipelineState {
|
|
|
79
78
|
|
|
80
79
|
// DopplerLoader instance
|
|
81
80
|
dopplerLoader: DopplerLoader | null;
|
|
81
|
+
ownsDopplerLoader: boolean;
|
|
82
82
|
|
|
83
83
|
// GPU context
|
|
84
84
|
gpuContext: { device?: GPUDevice } | null;
|
|
@@ -75,6 +75,7 @@ export class PipelineState {
|
|
|
75
75
|
// DopplerLoader instance
|
|
76
76
|
|
|
77
77
|
this.dopplerLoader = null;
|
|
78
|
+
this.ownsDopplerLoader = false;
|
|
78
79
|
|
|
79
80
|
// GPU context
|
|
80
81
|
|
|
@@ -111,6 +112,9 @@ export class PipelineState {
|
|
|
111
112
|
decodeReadbackCleanupMs: 0,
|
|
112
113
|
decodeReadbackCopyMs: 0,
|
|
113
114
|
prefillRecordMs: 0,
|
|
115
|
+
prefillRecordOps: 0,
|
|
116
|
+
prefillRecordPasses: 0,
|
|
117
|
+
prefillRecordOpLabels: {},
|
|
114
118
|
prefillSubmitWaitMs: 0,
|
|
115
119
|
prefillProfileSteps: [],
|
|
116
120
|
decodeProfileSteps: [],
|
|
@@ -46,6 +46,10 @@ export interface KVCacheSnapshot {
|
|
|
46
46
|
linearAttention?: LinearAttentionRuntime | null;
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
export interface WorkloadPhaseTiming {
|
|
50
|
+
[key: string]: number | string | boolean | null | Record<string, number> | WorkloadPhaseTiming | WorkloadPhaseTiming[];
|
|
51
|
+
}
|
|
52
|
+
|
|
49
53
|
export interface AdvanceEmbeddingResult {
|
|
50
54
|
embedding: Float32Array;
|
|
51
55
|
embeddingMode: 'last' | 'mean';
|
|
@@ -175,6 +179,12 @@ export interface LayerContext {
|
|
|
175
179
|
};
|
|
176
180
|
captureConfig?: Record<string, unknown>;
|
|
177
181
|
} | null;
|
|
182
|
+
/** Opt into fused gate/up GeGLU prefill path when runtime profile enables it. */
|
|
183
|
+
useFusedGateUpGelu?: boolean;
|
|
184
|
+
/** Opt into large-batch f16-weight/f32-activation fused gate/up prefill. */
|
|
185
|
+
useLargeBatchF16F32FusedGateUp?: boolean;
|
|
186
|
+
/** Skip KV cache writes for hidden-state-only prefill routes. */
|
|
187
|
+
skipKVCacheWrites?: boolean;
|
|
178
188
|
}
|
|
179
189
|
|
|
180
190
|
/**
|
|
@@ -408,6 +418,8 @@ export interface LogitsStepResult {
|
|
|
408
418
|
export interface PrefillResult extends KVCacheSnapshot {
|
|
409
419
|
/** Finalized logits for the next token after prefill */
|
|
410
420
|
logits: Float32Array;
|
|
421
|
+
|
|
422
|
+
phase?: WorkloadPhaseTiming | null;
|
|
411
423
|
}
|
|
412
424
|
|
|
413
425
|
/**
|
|
@@ -425,6 +437,8 @@ export interface PrefillEmbeddingResult extends KVCacheSnapshot {
|
|
|
425
437
|
|
|
426
438
|
/** Pooling mode used to construct embedding */
|
|
427
439
|
embeddingMode: 'last' | 'mean';
|
|
440
|
+
|
|
441
|
+
phase?: WorkloadPhaseTiming | null;
|
|
428
442
|
}
|
|
429
443
|
|
|
430
444
|
/**
|
|
@@ -485,6 +499,7 @@ export interface LayerWeights {
|
|
|
485
499
|
kProj: LayerWeightBuffer;
|
|
486
500
|
vProj?: LayerWeightBuffer;
|
|
487
501
|
oProj: LayerWeightBuffer;
|
|
502
|
+
qGateProj?: LayerWeightBuffer | null;
|
|
488
503
|
convInProj?: LayerWeightBuffer;
|
|
489
504
|
convKernel?: LayerWeightBuffer;
|
|
490
505
|
convOutProj?: LayerWeightBuffer;
|
|
@@ -492,8 +507,12 @@ export interface LayerWeights {
|
|
|
492
507
|
qkvProj?: GPUBuffer | WeightBuffer | null;
|
|
493
508
|
/** Sizes for splitting fused QKV output: [qSize, kSize, vSize] in elements */
|
|
494
509
|
qkvSizes?: [number, number, number];
|
|
495
|
-
/** Data type of fused QKV weights (f16 or
|
|
496
|
-
qkvDtype?: 'f16' | 'f32';
|
|
510
|
+
/** Data type of fused QKV weights (f16, f32, or q4k) */
|
|
511
|
+
qkvDtype?: 'f16' | 'f32' | 'q4k';
|
|
512
|
+
/** Fused linear-attention A/B projection weight for decode-only A+B projection fusion. */
|
|
513
|
+
linearABProj?: WeightBuffer | null;
|
|
514
|
+
/** Fused linear-attention QKV/Z projection weight for decode-only QKV+Z projection fusion. */
|
|
515
|
+
linearQKVZProj?: WeightBuffer | null;
|
|
497
516
|
|
|
498
517
|
// FFN (dense layers)
|
|
499
518
|
postAttentionNorm?: GPUBuffer | Float32Array;
|
|
@@ -610,6 +629,10 @@ export interface PipelineStats {
|
|
|
610
629
|
decodeRecordOps?: number;
|
|
611
630
|
decodeRecordPasses?: number;
|
|
612
631
|
decodeRecordOpLabels?: Record<string, number>;
|
|
632
|
+
prefillRecordMs?: number;
|
|
633
|
+
prefillRecordOps?: number;
|
|
634
|
+
prefillRecordPasses?: number;
|
|
635
|
+
prefillRecordOpLabels?: Record<string, number>;
|
|
613
636
|
uniformCache?: UniformCacheStats | null;
|
|
614
637
|
decodeSubmitWaitMs?: number;
|
|
615
638
|
decodeReadbackWaitMs?: number;
|
|
@@ -640,6 +663,7 @@ export interface PipelineStats {
|
|
|
640
663
|
activationDtype: string;
|
|
641
664
|
readbackInterval: number | null;
|
|
642
665
|
readbackMode: string | null;
|
|
666
|
+
maxBatchDecodeTokens?: number | null;
|
|
643
667
|
batchSize: number;
|
|
644
668
|
stopCheckMode: string;
|
|
645
669
|
disableCommandBatching?: boolean;
|
|
@@ -654,6 +678,7 @@ export interface PipelineStats {
|
|
|
654
678
|
activationDtype: string;
|
|
655
679
|
readbackInterval: number | null;
|
|
656
680
|
readbackMode: string | null;
|
|
681
|
+
maxBatchDecodeTokens?: number | null;
|
|
657
682
|
batchSize: number;
|
|
658
683
|
stopCheckMode: string;
|
|
659
684
|
disableCommandBatching?: boolean;
|
|
@@ -15,7 +15,7 @@ import { PipelineState } from './text/state.js';
|
|
|
15
15
|
import { PipelineGenerator } from './text/generator.js';
|
|
16
16
|
import type { Manifest } from './text/config.js';
|
|
17
17
|
import type { WeightLoadResult, PipelineContexts } from './text/init.js';
|
|
18
|
-
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats } from './text/types.js';
|
|
18
|
+
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming } from './text/types.js';
|
|
19
19
|
import type { ChatMessage } from './text/chat-format.js';
|
|
20
20
|
import type { LoRAAdapter } from './text/lora.js';
|
|
21
21
|
import type { DiffusionPipeline } from './diffusion/pipeline.js';
|
|
@@ -120,6 +120,7 @@ export declare class InferencePipeline extends PipelineState {
|
|
|
120
120
|
tokens: number[];
|
|
121
121
|
seqLen: number;
|
|
122
122
|
embeddingMode: string;
|
|
123
|
+
phase?: WorkloadPhaseTiming | null;
|
|
123
124
|
}>;
|
|
124
125
|
|
|
125
126
|
embedBatch(prompts: string[], options?: GenerateOptions): Promise<Array<{
|
|
@@ -131,6 +132,25 @@ export declare class InferencePipeline extends PipelineState {
|
|
|
131
132
|
|
|
132
133
|
prefillWithLogits(prompt: PromptInput, options?: GenerateOptions): Promise<PrefillResult>;
|
|
133
134
|
|
|
135
|
+
prefillWithTokenLogits(prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{
|
|
136
|
+
seqLen: number;
|
|
137
|
+
tokens: number[];
|
|
138
|
+
tokenIds: number[];
|
|
139
|
+
logits: Float32Array;
|
|
140
|
+
logitsByTokenId: Record<number, number>;
|
|
141
|
+
phase?: WorkloadPhaseTiming | null;
|
|
142
|
+
}>;
|
|
143
|
+
|
|
144
|
+
prefillWithTokenLogitsFromKV(prefix: KVCacheSnapshot, prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{
|
|
145
|
+
seqLen: number;
|
|
146
|
+
prefixTokens: number[];
|
|
147
|
+
tokens: number[];
|
|
148
|
+
tokenIds: number[];
|
|
149
|
+
logits: Float32Array;
|
|
150
|
+
logitsByTokenId: Record<number, number>;
|
|
151
|
+
phase?: WorkloadPhaseTiming | null;
|
|
152
|
+
}>;
|
|
153
|
+
|
|
134
154
|
applyKVCacheSnapshot(snapshot: KVCacheSnapshot): void;
|
|
135
155
|
|
|
136
156
|
generateWithPrefixKV(
|
|
@@ -299,6 +299,8 @@ export class InferencePipeline extends PipelineState {
|
|
|
299
299
|
assignStorageContext: true,
|
|
300
300
|
});
|
|
301
301
|
this.runtimeConfig = runtimeConfig;
|
|
302
|
+
this.dopplerLoader = contexts.loader || null;
|
|
303
|
+
this.ownsDopplerLoader = contexts.ownsLoader === true;
|
|
302
304
|
this.runtimeOverrides = contexts.runtimeConfig == null
|
|
303
305
|
? null
|
|
304
306
|
: (typeof structuredClone === 'function'
|
|
@@ -400,6 +402,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
400
402
|
numLayers: Number(manifest.architecture?.numLayers ?? 0),
|
|
401
403
|
capabilities,
|
|
402
404
|
platform,
|
|
405
|
+
useGPU: this.useGPU === true,
|
|
403
406
|
});
|
|
404
407
|
if (executionV1Runtime.executionV1State) {
|
|
405
408
|
this.runtimeConfig = executionV1Runtime.runtimeConfig;
|
|
@@ -648,6 +651,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
648
651
|
this.modelConfig.perLayerInputsSession ?? null,
|
|
649
652
|
this.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
650
653
|
),
|
|
654
|
+
loader: this.dopplerLoader ?? undefined,
|
|
651
655
|
onProgress: (info) => {
|
|
652
656
|
if (info.stage !== 'layers' && info.stage !== 'shards') {
|
|
653
657
|
log.verbose('Loader', `${info.stage}: ${Math.round(info.progress * 100)}%${info.message ? ` - ${info.message}` : ''}`);
|
|
@@ -675,7 +679,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
675
679
|
|
|
676
680
|
this.layerRouterWeights = result.layerRouterWeights;
|
|
677
681
|
|
|
678
|
-
this.dopplerLoader = getDopplerLoader(this.runtimeConfig.loading);
|
|
682
|
+
this.dopplerLoader = result.loader ?? getDopplerLoader(this.runtimeConfig.loading);
|
|
679
683
|
this.stats.loadTiming = result.loadTiming ?? this.dopplerLoader?.getLoadTiming?.() ?? null;
|
|
680
684
|
|
|
681
685
|
if ((this.modelConfig).useMoE && this.moeRouter) {
|
|
@@ -687,7 +691,11 @@ export class InferencePipeline extends PipelineState {
|
|
|
687
691
|
}
|
|
688
692
|
|
|
689
693
|
if (this.useGPU && this.modelConfig) {
|
|
690
|
-
|
|
694
|
+
const session = this.runtimeConfig?.inference?.session ?? null;
|
|
695
|
+
fuseQKVWeights(result.layerWeights, this.modelConfig, this.resolvedKernelPath, {
|
|
696
|
+
allowQ4K: session?.useFusedQKVSplitQKNorm === true
|
|
697
|
+
|| session?.useFusedQKVSplitQKNormRoPE === true,
|
|
698
|
+
});
|
|
691
699
|
}
|
|
692
700
|
|
|
693
701
|
if (this.useGPU && this.modelConfig) {
|
|
@@ -1577,17 +1585,23 @@ export class InferencePipeline extends PipelineState {
|
|
|
1577
1585
|
|
|
1578
1586
|
async embed(prompt, options = {}) {
|
|
1579
1587
|
assertNotAborted(options?.signal);
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1588
|
+
this.resetForBatch();
|
|
1589
|
+
try {
|
|
1590
|
+
const result = await this.prefillWithEmbedding(prompt, {
|
|
1591
|
+
...options,
|
|
1592
|
+
__skipStateSnapshot: true,
|
|
1593
|
+
});
|
|
1594
|
+
assertNotAborted(options?.signal);
|
|
1595
|
+
return {
|
|
1596
|
+
embedding: result.embedding,
|
|
1597
|
+
tokens: result.tokens,
|
|
1598
|
+
seqLen: result.seqLen,
|
|
1599
|
+
embeddingMode: result.embeddingMode,
|
|
1600
|
+
phase: result.phase ?? null,
|
|
1601
|
+
};
|
|
1602
|
+
} finally {
|
|
1603
|
+
this.resetForBatch();
|
|
1604
|
+
}
|
|
1591
1605
|
}
|
|
1592
1606
|
|
|
1593
1607
|
async embedBatch(prompts, options = {}) {
|
|
@@ -1601,7 +1615,6 @@ export class InferencePipeline extends PipelineState {
|
|
|
1601
1615
|
// Check between every prompt so a superseded revision drops the rest.
|
|
1602
1616
|
assertNotAborted(options?.signal);
|
|
1603
1617
|
outputs.push(await this.embed(prompt, batchOptions));
|
|
1604
|
-
this.resetForBatch();
|
|
1605
1618
|
}
|
|
1606
1619
|
return outputs;
|
|
1607
1620
|
}
|
|
@@ -1746,6 +1759,14 @@ export class InferencePipeline extends PipelineState {
|
|
|
1746
1759
|
return this.generator.prefillWithLogits(prompt, options);
|
|
1747
1760
|
}
|
|
1748
1761
|
|
|
1762
|
+
prefillWithTokenLogits(prompt, tokenIds, options = {}) {
|
|
1763
|
+
return this.generator.prefillWithTokenLogits(prompt, tokenIds, options);
|
|
1764
|
+
}
|
|
1765
|
+
|
|
1766
|
+
prefillWithTokenLogitsFromKV(prefix, prompt, tokenIds, options = {}) {
|
|
1767
|
+
return this.generator.prefillWithTokenLogitsFromKV(prefix, prompt, tokenIds, options);
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1749
1770
|
|
|
1750
1771
|
applyKVCacheSnapshot(snapshot) {
|
|
1751
1772
|
this.kvCache = snapshot.cache.clone();
|
|
@@ -1903,6 +1924,11 @@ export class InferencePipeline extends PipelineState {
|
|
|
1903
1924
|
this.plePrefetchPending = null;
|
|
1904
1925
|
this.weights.clear();
|
|
1905
1926
|
this.expertWeights.clear();
|
|
1927
|
+
if (this.ownsDopplerLoader && this.dopplerLoader) {
|
|
1928
|
+
await this.dopplerLoader.unload();
|
|
1929
|
+
}
|
|
1930
|
+
this.dopplerLoader = null;
|
|
1931
|
+
this.ownsDopplerLoader = false;
|
|
1906
1932
|
this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
|
|
1907
1933
|
this.lora = null;
|
|
1908
1934
|
destroyMoERouter(this.moeRouter);
|
|
@@ -118,6 +118,28 @@ function parseByteTokenValue(token) {
|
|
|
118
118
|
return hi >= 0 && lo >= 0 ? (hi << 4) | lo : null;
|
|
119
119
|
}
|
|
120
120
|
|
|
121
|
+
function decodeByteFallbackTokens(tokens) {
|
|
122
|
+
const parts = [];
|
|
123
|
+
let bytes = [];
|
|
124
|
+
const flushBytes = () => {
|
|
125
|
+
if (bytes.length === 0) return;
|
|
126
|
+
parts.push(new TextDecoder('utf-8', { fatal: false }).decode(Uint8Array.from(bytes)));
|
|
127
|
+
bytes = [];
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
for (const token of tokens) {
|
|
131
|
+
const byteValue = parseByteTokenValue(token);
|
|
132
|
+
if (byteValue !== null) {
|
|
133
|
+
bytes.push(byteValue);
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
flushBytes();
|
|
137
|
+
parts.push(token);
|
|
138
|
+
}
|
|
139
|
+
flushBytes();
|
|
140
|
+
return parts.join('');
|
|
141
|
+
}
|
|
142
|
+
|
|
121
143
|
function appendVocabEntry(token, id, vocab, reverseVocab, byteTokens) {
|
|
122
144
|
const numId = typeof id === 'number' ? id : parseInt( (id), 10);
|
|
123
145
|
vocab.set(token, numId);
|
|
@@ -469,9 +491,6 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
469
491
|
}
|
|
470
492
|
log.info('Tokenizer', `HuggingFace model.type="${model.type}", using type="${this.#type}"`);
|
|
471
493
|
this.#byteDecoder = null;
|
|
472
|
-
if (this.#type === 'bpe') {
|
|
473
|
-
this.#initializeByteDecoder();
|
|
474
|
-
}
|
|
475
494
|
let maxId = -1;
|
|
476
495
|
|
|
477
496
|
// Handle vocab based on type
|
|
@@ -586,6 +605,9 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
586
605
|
const decoderPrepend = hf.decoder?.prepend_scheme === 'always' || hf.decoder?.add_prefix_space === true;
|
|
587
606
|
const normalizerPrepend = hf.normalizer?.prepend_scheme === 'always' || hf.normalizer?.add_prefix_space === true;
|
|
588
607
|
this.#useByteLevelEncoding = byteLevelPretokenizer.useByteLevel;
|
|
608
|
+
if (this.#type === 'bpe' && this.#useByteLevelEncoding) {
|
|
609
|
+
this.#initializeByteDecoder();
|
|
610
|
+
}
|
|
589
611
|
const runtimeSpacePrefix = runtimeDefaults.addSpacePrefix;
|
|
590
612
|
// Use explicit runtime config if set (non-null), otherwise auto-detect from tokenizer.json
|
|
591
613
|
this.#addSpacePrefix = runtimeSpacePrefix
|
|
@@ -646,9 +668,6 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
646
668
|
throw new Error(`[Tokenizer] Unsupported tokenizer type: ${tokenizerJson.type}`);
|
|
647
669
|
}
|
|
648
670
|
this.#byteDecoder = null;
|
|
649
|
-
if (this.#type === 'bpe') {
|
|
650
|
-
this.#initializeByteDecoder();
|
|
651
|
-
}
|
|
652
671
|
|
|
653
672
|
// Build vocab maps
|
|
654
673
|
let maxId = loadObjectVocab(tokenizerJson.vocab, this.#vocab, this.#reverseVocab, this.#byteTokens);
|
|
@@ -744,6 +763,9 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
744
763
|
throw new Error('[Tokenizer] addEosToken is enabled but eos token is missing.');
|
|
745
764
|
}
|
|
746
765
|
this.#useByteLevelEncoding = byteLevelPretokenizer.useByteLevel;
|
|
766
|
+
if (this.#type === 'bpe' && this.#useByteLevelEncoding) {
|
|
767
|
+
this.#initializeByteDecoder();
|
|
768
|
+
}
|
|
747
769
|
// NOTE: Default to FALSE - first word shouldn't get space prefix
|
|
748
770
|
// Space prefixes are only for words that follow a space in original text
|
|
749
771
|
this.#addSpacePrefix = tokenizerJson.addSpacePrefix === true
|
|
@@ -1073,29 +1095,29 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
1073
1095
|
|
|
1074
1096
|
const token = this.#reverseVocab.get(id);
|
|
1075
1097
|
if (token !== undefined) {
|
|
1076
|
-
|
|
1077
|
-
if (token.match(/^<0x[0-9A-Fa-f]{2}>$/)) {
|
|
1078
|
-
const byteVal = parseInt(token.slice(3, 5), 16);
|
|
1079
|
-
tokens.push(String.fromCharCode(byteVal));
|
|
1080
|
-
} else {
|
|
1081
|
-
tokens.push(token);
|
|
1082
|
-
}
|
|
1098
|
+
tokens.push(token);
|
|
1083
1099
|
}
|
|
1084
1100
|
}
|
|
1085
1101
|
|
|
1086
1102
|
let result;
|
|
1087
|
-
if (this.#type === 'bpe' && this.#
|
|
1088
|
-
const merged = tokens.join('');
|
|
1103
|
+
if (this.#type === 'bpe' && this.#useByteLevelEncoding) {
|
|
1089
1104
|
const bytes = [];
|
|
1090
|
-
for (const
|
|
1091
|
-
const
|
|
1092
|
-
if (
|
|
1093
|
-
bytes.push(
|
|
1105
|
+
for (const token of tokens) {
|
|
1106
|
+
const byteValue = parseByteTokenValue(token);
|
|
1107
|
+
if (byteValue !== null) {
|
|
1108
|
+
bytes.push(byteValue);
|
|
1094
1109
|
continue;
|
|
1095
1110
|
}
|
|
1096
|
-
const
|
|
1097
|
-
|
|
1098
|
-
|
|
1111
|
+
for (const ch of token) {
|
|
1112
|
+
const mapped = this.#byteDecoder.get(ch);
|
|
1113
|
+
if (mapped != null) {
|
|
1114
|
+
bytes.push(mapped);
|
|
1115
|
+
continue;
|
|
1116
|
+
}
|
|
1117
|
+
const fallbackBytes = new TextEncoder().encode(ch);
|
|
1118
|
+
for (const byte of fallbackBytes) {
|
|
1119
|
+
bytes.push(byte);
|
|
1120
|
+
}
|
|
1099
1121
|
}
|
|
1100
1122
|
}
|
|
1101
1123
|
result = new TextDecoder('utf-8', { fatal: false }).decode(Uint8Array.from(bytes));
|
|
@@ -1103,7 +1125,7 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
1103
1125
|
result = result.replace(/▁/g, ' ');
|
|
1104
1126
|
} else {
|
|
1105
1127
|
// Join and convert ▁ back to spaces, handle GPT-style markers
|
|
1106
|
-
result = tokens
|
|
1128
|
+
result = decodeByteFallbackTokens(tokens)
|
|
1107
1129
|
.replace(/▁/g, ' ')
|
|
1108
1130
|
.replace(/Ġ/g, ' ')
|
|
1109
1131
|
.replace(/Ċ/g, '\n');
|
|
@@ -4,9 +4,13 @@
|
|
|
4
4
|
* @module inference/tokenizers/types
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
export interface SpecialTokens {
|
|
8
|
+
bos?: number;
|
|
9
|
+
eos?: number;
|
|
10
|
+
pad?: number;
|
|
11
|
+
unk?: number;
|
|
12
|
+
mask?: number;
|
|
13
|
+
}
|
|
10
14
|
|
|
11
15
|
/** Tokenizer Configuration */
|
|
12
16
|
export interface TokenizerConfig {
|
|
@@ -144,4 +148,3 @@ export interface BundledTokenizerJson {
|
|
|
144
148
|
addEosToken?: boolean;
|
|
145
149
|
addSpacePrefix?: boolean;
|
|
146
150
|
}
|
|
147
|
-
|