doppler-gpu 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -0,0 +1,1581 @@
|
|
|
1
|
+
import {
|
|
2
|
+
createWeightBuffer,
|
|
3
|
+
getBufferDtype,
|
|
4
|
+
getLayout,
|
|
5
|
+
getWeightDtype,
|
|
6
|
+
isCpuWeightBuffer,
|
|
7
|
+
isGpuBufferInstance,
|
|
8
|
+
isWeightBuffer,
|
|
9
|
+
} from '../../../gpu/weight-buffer.js';
|
|
10
|
+
import { createTensor } from '../../../gpu/tensor.js';
|
|
11
|
+
import { recordScale, runScale } from '../../../gpu/kernel-selector.js';
|
|
12
|
+
import { getNormWeightBuffer, getWeightBuffer } from './weights.js';
|
|
13
|
+
import { doMatmul, doRMSNorm, releaseOrTrack } from './ops.js';
|
|
14
|
+
import { runProbes } from './probes.js';
|
|
15
|
+
import { embed, isRangeBackedCpuEmbeddingSource, normalizeRangeBytes, decodeRangeChunkIntoOutput } from './embed.js';
|
|
16
|
+
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
17
|
+
import { getDevice } from '../../../gpu/device.js';
|
|
18
|
+
import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
19
|
+
|
|
20
|
+
const pleRuntimeCache = new WeakMap();
|
|
21
|
+
const pleRangeRowCache = new WeakMap();
|
|
22
|
+
|
|
23
|
+
function getPerLayerInputWeights(context) {
|
|
24
|
+
const weights = context.weights.get('per_layer_inputs');
|
|
25
|
+
if (!weights || typeof weights !== 'object') {
|
|
26
|
+
throw new Error(
|
|
27
|
+
'Gemma 4 per-layer inputs require global per-layer input weights, ' +
|
|
28
|
+
'but state.weights.get("per_layer_inputs") was missing.'
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
return weights;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function normalizePleProjectionNormDtype(dtype) {
|
|
35
|
+
if (typeof dtype !== 'string') {
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
const value = dtype.toLowerCase();
|
|
39
|
+
if (value === 'f16' || value === 'f32') {
|
|
40
|
+
return value;
|
|
41
|
+
}
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function getPleProjectionNormDtype(weight) {
|
|
46
|
+
return normalizePleProjectionNormDtype(getWeightDtype(weight))
|
|
47
|
+
?? normalizePleProjectionNormDtype(getBufferDtype(weight))
|
|
48
|
+
?? null;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function isRangeBackedPleProjectionSource(value) {
|
|
52
|
+
return isRangeBackedCpuEmbeddingSource(value);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function getPleProjectionWeightDtype(weight) {
|
|
56
|
+
if (isCpuWeightBuffer(weight)) {
|
|
57
|
+
return weight.dtype ?? null;
|
|
58
|
+
}
|
|
59
|
+
return getWeightDtype(weight);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function isDensePleProjectionDtype(dtype) {
|
|
63
|
+
return dtype === 'f16' || dtype === 'f32';
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function resolveDensePleProjectionWeight(
|
|
67
|
+
weight,
|
|
68
|
+
label = 'per_layer_model_projection'
|
|
69
|
+
) {
|
|
70
|
+
if (isWeightBuffer(weight)) {
|
|
71
|
+
const dtype = String(weight.dtype ?? '').toLowerCase();
|
|
72
|
+
if (!isDensePleProjectionDtype(dtype)) {
|
|
73
|
+
throw new Error(
|
|
74
|
+
'Gemma 4 sliced per-layer projection requires a dense f16/f32 base materialization. ' +
|
|
75
|
+
`Got dtype="${String(weight.dtype)}" for "${label}".`
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
if (!weight.materializations?.q4k?.buffer) {
|
|
79
|
+
return weight;
|
|
80
|
+
}
|
|
81
|
+
return createWeightBuffer(
|
|
82
|
+
weight.buffer,
|
|
83
|
+
dtype,
|
|
84
|
+
weight.layout,
|
|
85
|
+
[...weight.shape],
|
|
86
|
+
weight.label ?? label
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
if (isGpuBufferInstance(weight)) {
|
|
91
|
+
const dtype = String(getBufferDtype(weight) ?? '').toLowerCase();
|
|
92
|
+
if (!isDensePleProjectionDtype(dtype)) {
|
|
93
|
+
throw new Error(
|
|
94
|
+
'Gemma 4 sliced per-layer projection requires dense f16/f32 GPU weights with runtime dtype metadata. ' +
|
|
95
|
+
`Got dtype="${String(getBufferDtype(weight) ?? 'unknown')}" for "${label}".`
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return weight;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export async function loadRangeBackedPleProjectionSliceBytes(
|
|
104
|
+
weight,
|
|
105
|
+
layerIdx,
|
|
106
|
+
hiddenSizePerLayerInput,
|
|
107
|
+
hiddenSize,
|
|
108
|
+
label = 'Range-backed PLE projection slice'
|
|
109
|
+
) {
|
|
110
|
+
if (!isCpuWeightBuffer(weight)) {
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
const cpuData = weight.data;
|
|
114
|
+
if (!isRangeBackedPleProjectionSource(cpuData)) {
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const dtype = getPleProjectionWeightDtype(weight);
|
|
119
|
+
const layout = getLayout(weight) ?? weight.layout ?? 'row';
|
|
120
|
+
const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', {
|
|
121
|
+
dtype,
|
|
122
|
+
});
|
|
123
|
+
const hiddenOffset = layerIdx * hiddenSizePerLayerInput;
|
|
124
|
+
const byteOffset = hiddenOffset * hiddenSize * bytesPerElement;
|
|
125
|
+
const byteLength = hiddenSizePerLayerInput * hiddenSize * bytesPerElement;
|
|
126
|
+
const bytes = normalizeRangeBytes(
|
|
127
|
+
await cpuData.loadRange(byteOffset, byteLength),
|
|
128
|
+
label
|
|
129
|
+
);
|
|
130
|
+
if (bytes.byteLength !== byteLength) {
|
|
131
|
+
throw new Error(
|
|
132
|
+
`${label} short read for layer ${layerIdx}: expected ${byteLength} bytes, got ${bytes.byteLength}.`
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
return {
|
|
136
|
+
bytes,
|
|
137
|
+
dtype,
|
|
138
|
+
layout,
|
|
139
|
+
shape: [hiddenSizePerLayerInput, hiddenSize],
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function inferPleProjectionNormDtype(weight, hiddenSizePerLayerInput) {
|
|
144
|
+
const explicitDtype = getPleProjectionNormDtype(weight);
|
|
145
|
+
if (explicitDtype) {
|
|
146
|
+
return explicitDtype;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if (!isGpuBufferInstance(weight)) {
|
|
150
|
+
return 'f32';
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
|
|
154
|
+
throw new Error('Gemma 4 per-layer projection norm dtype inference requires hiddenSizePerLayerInput > 0.');
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const bytesPerElement = weight.size / hiddenSizePerLayerInput;
|
|
158
|
+
if (bytesPerElement !== 2 && bytesPerElement !== 4) {
|
|
159
|
+
throw new Error(
|
|
160
|
+
'Gemma 4 per-layer projection norm buffer has unsupported byte size: ' +
|
|
161
|
+
`bufferSize=${weight.size}, hiddenSizePerLayerInput=${hiddenSizePerLayerInput}.`
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
return selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement });
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function createPleProjectionNormTensor(buffer, dtype, hiddenSizePerLayerInput, label) {
|
|
168
|
+
return createTensor(buffer, dtype, [hiddenSizePerLayerInput], label);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function destroyPleRuntimeCacheEntry(entry) {
|
|
172
|
+
const cachedBuffer = entry?.scaledProjectionNormWeight?.buffer ?? null;
|
|
173
|
+
if (cachedBuffer) {
|
|
174
|
+
releaseBuffer(cachedBuffer);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function destroyPleSplitTables(perLayerInputWeights) {
|
|
179
|
+
const splitTables = Array.isArray(perLayerInputWeights?.embedTokensPerLayerSplit)
|
|
180
|
+
? perLayerInputWeights.embedTokensPerLayerSplit
|
|
181
|
+
: null;
|
|
182
|
+
if (!splitTables) {
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
for (const table of splitTables) {
|
|
186
|
+
const buffer = table?.buffer ?? table ?? null;
|
|
187
|
+
if (buffer) {
|
|
188
|
+
releaseBuffer(buffer);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
delete perLayerInputWeights.embedTokensPerLayerSplit;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function destroyPleHotVocabularyRuntime(perLayerInputWeights) {
|
|
195
|
+
const runtime = perLayerInputWeights?.embedTokensPerLayerHotRuntime ?? null;
|
|
196
|
+
if (!runtime || typeof runtime !== 'object') {
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
const splitTables = Array.isArray(runtime.splitTables) ? runtime.splitTables : [];
|
|
200
|
+
for (const table of splitTables) {
|
|
201
|
+
const buffer = table?.buffer ?? table ?? null;
|
|
202
|
+
if (buffer) {
|
|
203
|
+
releaseBuffer(buffer);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
const mapBuffer = runtime.hotTokenIndexMapBuffer ?? null;
|
|
207
|
+
if (mapBuffer) {
|
|
208
|
+
releaseBuffer(mapBuffer);
|
|
209
|
+
}
|
|
210
|
+
delete perLayerInputWeights.embedTokensPerLayerHotRuntime;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
export function destroyPleRuntimeCache(perLayerInputWeights) {
|
|
214
|
+
if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
const entry = pleRuntimeCache.get(perLayerInputWeights);
|
|
218
|
+
if (entry) {
|
|
219
|
+
destroyPleRuntimeCacheEntry(entry);
|
|
220
|
+
pleRuntimeCache.delete(perLayerInputWeights);
|
|
221
|
+
}
|
|
222
|
+
destroyPleSplitTables(perLayerInputWeights);
|
|
223
|
+
destroyPleHotVocabularyRuntime(perLayerInputWeights);
|
|
224
|
+
if (perLayerInputWeights.embedTokensPerLayer) {
|
|
225
|
+
pleRangeRowCache.delete(perLayerInputWeights.embedTokensPerLayer);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export function scalePerLayerProjectionNormWeights(weight, combineScale, rmsNormWeightOffset = false) {
|
|
230
|
+
if (rmsNormWeightOffset) {
|
|
231
|
+
return null;
|
|
232
|
+
}
|
|
233
|
+
const source = isCpuWeightBuffer(weight) ? weight.data : weight;
|
|
234
|
+
const isArrayLikeView = ArrayBuffer.isView(source) && typeof source.length === 'number';
|
|
235
|
+
if (!(source instanceof Float32Array) && !isArrayLikeView && !Array.isArray(source)) {
|
|
236
|
+
return null;
|
|
237
|
+
}
|
|
238
|
+
const scaled = Float32Array.from(source);
|
|
239
|
+
for (let i = 0; i < scaled.length; i++) {
|
|
240
|
+
scaled[i] *= combineScale;
|
|
241
|
+
}
|
|
242
|
+
return scaled;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
export async function ensurePleScaledProjectionNormWeight(context, combineScale = 2 ** -0.5) {
|
|
246
|
+
const hiddenSizePerLayerInput = Number(context?.config?.hiddenSizePerLayerInput ?? 0);
|
|
247
|
+
if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
|
|
248
|
+
return null;
|
|
249
|
+
}
|
|
250
|
+
if (context?.weightConfig?.rmsNormWeightOffset) {
|
|
251
|
+
return null;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const perLayerInputWeights = getPerLayerInputWeights(context);
|
|
255
|
+
const cachedEntry = pleRuntimeCache.get(perLayerInputWeights) ?? null;
|
|
256
|
+
if (cachedEntry?.combineScale === combineScale && cachedEntry.scaledProjectionNormWeight) {
|
|
257
|
+
return cachedEntry.scaledProjectionNormWeight;
|
|
258
|
+
}
|
|
259
|
+
if (cachedEntry) {
|
|
260
|
+
destroyPleRuntimeCacheEntry(cachedEntry);
|
|
261
|
+
pleRuntimeCache.delete(perLayerInputWeights);
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
const projectionNormWeight = perLayerInputWeights.perLayerProjectionNorm;
|
|
265
|
+
if (!projectionNormWeight) {
|
|
266
|
+
return null;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
let scaledProjectionNormWeight = null;
|
|
270
|
+
if (isGpuBufferInstance(projectionNormWeight)) {
|
|
271
|
+
const projectionNormDtype = inferPleProjectionNormDtype(
|
|
272
|
+
projectionNormWeight,
|
|
273
|
+
hiddenSizePerLayerInput
|
|
274
|
+
);
|
|
275
|
+
const scaledTensor = await runScale(
|
|
276
|
+
createPleProjectionNormTensor(
|
|
277
|
+
projectionNormWeight,
|
|
278
|
+
projectionNormDtype,
|
|
279
|
+
hiddenSizePerLayerInput,
|
|
280
|
+
'per_layer_projection_norm'
|
|
281
|
+
),
|
|
282
|
+
combineScale,
|
|
283
|
+
{ count: hiddenSizePerLayerInput }
|
|
284
|
+
);
|
|
285
|
+
scaledProjectionNormWeight = createPleProjectionNormTensor(
|
|
286
|
+
scaledTensor.buffer,
|
|
287
|
+
projectionNormDtype,
|
|
288
|
+
hiddenSizePerLayerInput,
|
|
289
|
+
'per_layer_projection_norm_scaled'
|
|
290
|
+
);
|
|
291
|
+
} else {
|
|
292
|
+
const scaledValues = scalePerLayerProjectionNormWeights(
|
|
293
|
+
projectionNormWeight,
|
|
294
|
+
combineScale,
|
|
295
|
+
false
|
|
296
|
+
);
|
|
297
|
+
if (!scaledValues) {
|
|
298
|
+
return null;
|
|
299
|
+
}
|
|
300
|
+
if (scaledValues.length !== hiddenSizePerLayerInput) {
|
|
301
|
+
throw new Error(
|
|
302
|
+
'Gemma 4 per-layer projection norm cache shape mismatch: ' +
|
|
303
|
+
`expected ${hiddenSizePerLayerInput} values, got ${scaledValues.length}.`
|
|
304
|
+
);
|
|
305
|
+
}
|
|
306
|
+
const device = getDevice();
|
|
307
|
+
if (!device) {
|
|
308
|
+
throw new Error('No GPU device available for Gemma 4 per-layer projection norm cache.');
|
|
309
|
+
}
|
|
310
|
+
const scaledBuffer = acquireBuffer(
|
|
311
|
+
scaledValues.byteLength,
|
|
312
|
+
undefined,
|
|
313
|
+
'per_layer_projection_norm_scaled'
|
|
314
|
+
);
|
|
315
|
+
try {
|
|
316
|
+
device.queue.writeBuffer(scaledBuffer, 0, scaledValues);
|
|
317
|
+
} catch (error) {
|
|
318
|
+
releaseBuffer(scaledBuffer);
|
|
319
|
+
throw error;
|
|
320
|
+
}
|
|
321
|
+
scaledProjectionNormWeight = createPleProjectionNormTensor(
|
|
322
|
+
scaledBuffer,
|
|
323
|
+
getPleProjectionNormDtype(projectionNormWeight) ?? 'f32',
|
|
324
|
+
hiddenSizePerLayerInput,
|
|
325
|
+
'per_layer_projection_norm_scaled'
|
|
326
|
+
);
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
if (scaledProjectionNormWeight?.shape?.[0] !== hiddenSizePerLayerInput) {
|
|
330
|
+
throw new Error(
|
|
331
|
+
'Gemma 4 per-layer projection norm cache shape mismatch after tensor creation: ' +
|
|
332
|
+
`expected ${hiddenSizePerLayerInput}, got ${scaledProjectionNormWeight?.shape?.[0] ?? 'unknown'}.`
|
|
333
|
+
);
|
|
334
|
+
}
|
|
335
|
+
if (!normalizePleProjectionNormDtype(scaledProjectionNormWeight?.dtype)) {
|
|
336
|
+
throw new Error(
|
|
337
|
+
'Gemma 4 per-layer projection norm cache produced an invalid dtype: ' +
|
|
338
|
+
`"${String(scaledProjectionNormWeight?.dtype ?? 'undefined')}".`
|
|
339
|
+
);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
pleRuntimeCache.set(perLayerInputWeights, {
|
|
343
|
+
combineScale,
|
|
344
|
+
scaledProjectionNormWeight,
|
|
345
|
+
});
|
|
346
|
+
return scaledProjectionNormWeight;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function getEmbeddingSource(weight, label) {
|
|
350
|
+
if (isWeightBuffer(weight)) {
|
|
351
|
+
return weight.buffer;
|
|
352
|
+
}
|
|
353
|
+
if (isCpuWeightBuffer(weight) || isGpuBufferInstance(weight) || weight instanceof Float32Array) {
|
|
354
|
+
return weight;
|
|
355
|
+
}
|
|
356
|
+
throw new Error(`Gemma 4 per-layer input ${label} has unsupported type "${weight?.constructor?.name ?? typeof weight}".`);
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
function getEmbeddingDtype(weight) {
|
|
360
|
+
if (isCpuWeightBuffer(weight)) {
|
|
361
|
+
return weight.dtype;
|
|
362
|
+
}
|
|
363
|
+
return getWeightDtype(weight);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
function getEmbeddingTranspose(weight) {
|
|
367
|
+
if (isWeightBuffer(weight) || isCpuWeightBuffer(weight)) {
|
|
368
|
+
return weight.layout === 'column';
|
|
369
|
+
}
|
|
370
|
+
return false;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// Step 4: Pre-allocated buffer cache for decode-path fused projection slices.
|
|
374
|
+
// Avoids per-step acquireBuffer/releaseBuffer churn for the 35 slice buffers.
|
|
375
|
+
export function createPleBufferCache(numLayers, sliceBytes) {
|
|
376
|
+
const sliceBuffers = Array.from({ length: numLayers }, (_, l) =>
|
|
377
|
+
acquireBuffer(sliceBytes, undefined, `L${l}.ple_slice_cached`));
|
|
378
|
+
const gatherSliceBuffers = Array.from({ length: numLayers }, (_, l) =>
|
|
379
|
+
acquireBuffer(sliceBytes, undefined, `L${l}.ple_gather_slice_cached`));
|
|
380
|
+
const ownedBuffers = new Set();
|
|
381
|
+
for (const buffer of sliceBuffers) {
|
|
382
|
+
if (buffer) ownedBuffers.add(buffer);
|
|
383
|
+
}
|
|
384
|
+
for (const buffer of gatherSliceBuffers) {
|
|
385
|
+
if (buffer) ownedBuffers.add(buffer);
|
|
386
|
+
}
|
|
387
|
+
return {
|
|
388
|
+
sliceBuffers,
|
|
389
|
+
gatherSliceBuffers,
|
|
390
|
+
preparedTokenEntries: new Map(),
|
|
391
|
+
preparedTokenBytes: 0,
|
|
392
|
+
ownedBuffers,
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
export function destroyPleBufferCache(cache) {
|
|
397
|
+
if (!cache?.sliceBuffers && !cache?.gatherSliceBuffers) return;
|
|
398
|
+
for (const buf of cache?.sliceBuffers ?? []) {
|
|
399
|
+
if (buf) releaseBuffer(buf);
|
|
400
|
+
}
|
|
401
|
+
for (const buf of cache?.gatherSliceBuffers ?? []) {
|
|
402
|
+
if (buf) releaseBuffer(buf);
|
|
403
|
+
}
|
|
404
|
+
for (const entry of cache?.preparedTokenEntries?.values?.() ?? []) {
|
|
405
|
+
for (const buf of entry?.buffers ?? []) {
|
|
406
|
+
if (buf) releaseBuffer(buf);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
cache.sliceBuffers = null;
|
|
410
|
+
cache.gatherSliceBuffers = null;
|
|
411
|
+
cache.preparedTokenEntries = null;
|
|
412
|
+
cache.preparedTokenBytes = 0;
|
|
413
|
+
cache.ownedBuffers = null;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
function isCachedPleSliceBuffer(cache, buffer) {
|
|
417
|
+
return cache?.ownedBuffers instanceof Set && cache.ownedBuffers.has(buffer);
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
function releasePleSliceBuffer(recorder, buffer, decodeBuffers, cache) {
|
|
421
|
+
if (!buffer || isCachedPleSliceBuffer(cache, buffer)) {
|
|
422
|
+
return;
|
|
423
|
+
}
|
|
424
|
+
releaseOrTrack(recorder, buffer, decodeBuffers);
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
function getPleRowCachePolicy(sessionConfig) {
|
|
428
|
+
const rowCache = sessionConfig?.rowCache ?? null;
|
|
429
|
+
if (!rowCache || rowCache.mode === 'off') {
|
|
430
|
+
return null;
|
|
431
|
+
}
|
|
432
|
+
if (rowCache.mode !== 'lru') {
|
|
433
|
+
throw new Error(
|
|
434
|
+
`Gemma 4 per-layer input row cache mode "${String(rowCache.mode)}" is not implemented.`
|
|
435
|
+
);
|
|
436
|
+
}
|
|
437
|
+
const decodedDtype = String(rowCache.decodedDtype ?? '').toLowerCase();
|
|
438
|
+
if (decodedDtype !== 'f32') {
|
|
439
|
+
throw new Error(
|
|
440
|
+
`Gemma 4 range-backed per-layer input row cache requires rowCache.decodedDtype="f32"; ` +
|
|
441
|
+
`got "${String(rowCache.decodedDtype)}".`
|
|
442
|
+
);
|
|
443
|
+
}
|
|
444
|
+
const maxRows = Math.trunc(Number(rowCache.maxRows));
|
|
445
|
+
const maxBytes = Math.trunc(Number(rowCache.maxBytes));
|
|
446
|
+
if (!Number.isFinite(maxRows) || maxRows <= 0) {
|
|
447
|
+
throw new Error('Gemma 4 per-layer input row cache requires rowCache.maxRows > 0.');
|
|
448
|
+
}
|
|
449
|
+
if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
|
|
450
|
+
throw new Error('Gemma 4 per-layer input row cache requires rowCache.maxBytes > 0.');
|
|
451
|
+
}
|
|
452
|
+
return { maxRows, maxBytes };
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
function getPleHotCachePolicy(sessionConfig) {
|
|
456
|
+
const hotCache = sessionConfig?.hotCache ?? null;
|
|
457
|
+
if (!hotCache || hotCache.mode === 'off') {
|
|
458
|
+
return null;
|
|
459
|
+
}
|
|
460
|
+
if (hotCache.mode === 'tokenizer_scores') {
|
|
461
|
+
const outputDtype = String(hotCache.outputDtype ?? '').toLowerCase();
|
|
462
|
+
if (outputDtype !== 'f16' && outputDtype !== 'f32') {
|
|
463
|
+
throw new Error(
|
|
464
|
+
`Gemma 4 per-layer input hot vocabulary cache requires hotCache.outputDtype to be "f16" or "f32"; ` +
|
|
465
|
+
`got "${String(hotCache.outputDtype)}".`
|
|
466
|
+
);
|
|
467
|
+
}
|
|
468
|
+
const maxTokens = Math.trunc(Number(hotCache.maxTokens));
|
|
469
|
+
const maxBytes = Math.trunc(Number(hotCache.maxBytes));
|
|
470
|
+
if (!Number.isFinite(maxTokens) || maxTokens <= 0) {
|
|
471
|
+
throw new Error('Gemma 4 per-layer input hot vocabulary cache requires hotCache.maxTokens > 0.');
|
|
472
|
+
}
|
|
473
|
+
if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
|
|
474
|
+
throw new Error('Gemma 4 per-layer input hot vocabulary cache requires hotCache.maxBytes > 0.');
|
|
475
|
+
}
|
|
476
|
+
return {
|
|
477
|
+
mode: 'tokenizer_scores',
|
|
478
|
+
maxTokens,
|
|
479
|
+
maxBytes,
|
|
480
|
+
outputDtype,
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
if (hotCache.mode !== 'prepared_tokens') {
|
|
484
|
+
throw new Error(
|
|
485
|
+
`Gemma 4 per-layer input hot cache mode "${String(hotCache.mode)}" is not implemented.`
|
|
486
|
+
);
|
|
487
|
+
}
|
|
488
|
+
const outputDtype = String(hotCache.outputDtype ?? '').toLowerCase();
|
|
489
|
+
if (outputDtype !== 'f16' && outputDtype !== 'f32') {
|
|
490
|
+
throw new Error(
|
|
491
|
+
`Gemma 4 per-layer input hot cache requires hotCache.outputDtype to be "f16" or "f32"; ` +
|
|
492
|
+
`got "${String(hotCache.outputDtype)}".`
|
|
493
|
+
);
|
|
494
|
+
}
|
|
495
|
+
const maxTokens = Math.trunc(Number(hotCache.maxTokens));
|
|
496
|
+
const maxBytes = Math.trunc(Number(hotCache.maxBytes));
|
|
497
|
+
if (!Number.isFinite(maxTokens) || maxTokens <= 0) {
|
|
498
|
+
throw new Error('Gemma 4 per-layer input hot cache requires hotCache.maxTokens > 0.');
|
|
499
|
+
}
|
|
500
|
+
if (!Number.isFinite(maxBytes) || maxBytes <= 0) {
|
|
501
|
+
throw new Error('Gemma 4 per-layer input hot cache requires hotCache.maxBytes > 0.');
|
|
502
|
+
}
|
|
503
|
+
return { mode: 'prepared_tokens', maxTokens, maxBytes, outputDtype };
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
export function getPleHotVocabularyRuntime(context) {
|
|
507
|
+
const perLayerInputWeights = context?.weights?.get?.('per_layer_inputs');
|
|
508
|
+
if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
|
|
509
|
+
return null;
|
|
510
|
+
}
|
|
511
|
+
const runtime = perLayerInputWeights.embedTokensPerLayerHotRuntime ?? null;
|
|
512
|
+
return runtime && typeof runtime === 'object' ? runtime : null;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
function getPleSplitTablePolicy(sessionConfig) {
|
|
516
|
+
if (sessionConfig?.materialization !== 'gpu_split_tables') {
|
|
517
|
+
return null;
|
|
518
|
+
}
|
|
519
|
+
return { mode: 'gpu_split_tables' };
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function releasePreparedTokenEntry(cache, tokenId, entry) {
|
|
523
|
+
if (!cache || !entry) {
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
cache.preparedTokenEntries?.delete(tokenId);
|
|
527
|
+
cache.preparedTokenBytes -= entry?.bytes ?? 0;
|
|
528
|
+
for (const buffer of entry.buffers ?? []) {
|
|
529
|
+
if (!buffer) continue;
|
|
530
|
+
cache.ownedBuffers?.delete(buffer);
|
|
531
|
+
releaseBuffer(buffer);
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
function prunePreparedTokenCache(cache, policy) {
|
|
536
|
+
while (
|
|
537
|
+
cache.preparedTokenEntries.size > policy.maxTokens
|
|
538
|
+
|| cache.preparedTokenBytes > policy.maxBytes
|
|
539
|
+
) {
|
|
540
|
+
const oldest = cache.preparedTokenEntries.keys().next();
|
|
541
|
+
if (oldest.done) {
|
|
542
|
+
break;
|
|
543
|
+
}
|
|
544
|
+
const tokenId = oldest.value;
|
|
545
|
+
const entry = cache.preparedTokenEntries.get(tokenId);
|
|
546
|
+
releasePreparedTokenEntry(cache, tokenId, entry);
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
function getPreparedTokenEntry(cache, tokenId, sessionConfig, activationDtype, stats = null) {
|
|
551
|
+
const policy = getPleHotCachePolicy(sessionConfig);
|
|
552
|
+
if (!policy || policy.mode !== 'prepared_tokens' || !(cache?.preparedTokenEntries instanceof Map)) {
|
|
553
|
+
return null;
|
|
554
|
+
}
|
|
555
|
+
if (policy.outputDtype !== activationDtype) {
|
|
556
|
+
throw new Error(
|
|
557
|
+
`Gemma 4 prepared per-layer input hot cache requires activation dtype "${policy.outputDtype}", ` +
|
|
558
|
+
`got "${String(activationDtype)}".`
|
|
559
|
+
);
|
|
560
|
+
}
|
|
561
|
+
const entry = cache.preparedTokenEntries.get(tokenId) ?? null;
|
|
562
|
+
if (!entry) {
|
|
563
|
+
if (stats) {
|
|
564
|
+
stats.plePreparedTokenCacheMisses = (stats.plePreparedTokenCacheMisses ?? 0) + 1;
|
|
565
|
+
stats.plePreparedTokenCacheEntries = cache.preparedTokenEntries.size;
|
|
566
|
+
stats.plePreparedTokenCacheBytes = cache.preparedTokenBytes;
|
|
567
|
+
}
|
|
568
|
+
return null;
|
|
569
|
+
}
|
|
570
|
+
cache.preparedTokenEntries.delete(tokenId);
|
|
571
|
+
cache.preparedTokenEntries.set(tokenId, entry);
|
|
572
|
+
if (stats) {
|
|
573
|
+
stats.plePreparedTokenCacheHits = (stats.plePreparedTokenCacheHits ?? 0) + 1;
|
|
574
|
+
stats.plePreparedTokenCacheEntries = cache.preparedTokenEntries.size;
|
|
575
|
+
stats.plePreparedTokenCacheBytes = cache.preparedTokenBytes;
|
|
576
|
+
}
|
|
577
|
+
return entry.buffers.slice();
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
function storePreparedTokenEntry(cache, tokenId, buffers, sessionConfig, activationDtype, stats = null) {
|
|
581
|
+
const policy = getPleHotCachePolicy(sessionConfig);
|
|
582
|
+
if (!policy || policy.mode !== 'prepared_tokens' || !(cache?.preparedTokenEntries instanceof Map)) {
|
|
583
|
+
return buffers;
|
|
584
|
+
}
|
|
585
|
+
if (policy.outputDtype !== activationDtype) {
|
|
586
|
+
throw new Error(
|
|
587
|
+
`Gemma 4 prepared per-layer input hot cache requires activation dtype "${policy.outputDtype}", ` +
|
|
588
|
+
`got "${String(activationDtype)}".`
|
|
589
|
+
);
|
|
590
|
+
}
|
|
591
|
+
const existing = cache.preparedTokenEntries.get(tokenId) ?? null;
|
|
592
|
+
if (existing) {
|
|
593
|
+
releasePreparedTokenEntry(cache, tokenId, existing);
|
|
594
|
+
}
|
|
595
|
+
const cachedBuffers = buffers.slice();
|
|
596
|
+
const bytes = cachedBuffers.reduce((total, buffer) => total + (buffer?.size ?? 0), 0);
|
|
597
|
+
for (const buffer of cachedBuffers) {
|
|
598
|
+
if (buffer) {
|
|
599
|
+
cache.ownedBuffers?.add(buffer);
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
cache.preparedTokenEntries.set(tokenId, { buffers: cachedBuffers, bytes });
|
|
603
|
+
cache.preparedTokenBytes += bytes;
|
|
604
|
+
prunePreparedTokenCache(cache, policy);
|
|
605
|
+
if (stats) {
|
|
606
|
+
stats.plePreparedTokenCacheEntries = cache.preparedTokenEntries.size;
|
|
607
|
+
stats.plePreparedTokenCacheBytes = cache.preparedTokenBytes;
|
|
608
|
+
}
|
|
609
|
+
return cachedBuffers.slice();
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
function getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize) {
|
|
613
|
+
const sourceDtype = String(
|
|
614
|
+
(isCpuWeightBuffer(embedTokensPerLayer) ? embedTokensPerLayer.data?.sourceDtype : null)
|
|
615
|
+
?? embedTokensPerLayer?.dtype
|
|
616
|
+
?? 'f32'
|
|
617
|
+
).toLowerCase();
|
|
618
|
+
const bytesPerElement = (sourceDtype === 'f16' || sourceDtype === 'bf16') ? 2 : 4;
|
|
619
|
+
return {
|
|
620
|
+
sourceDtype,
|
|
621
|
+
sourceRowBytes: totalPerLayerHiddenSize * bytesPerElement,
|
|
622
|
+
};
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
function getPleRangeRowCache(embedTokensPerLayer, sessionConfig) {
|
|
626
|
+
const policy = getPleRowCachePolicy(sessionConfig);
|
|
627
|
+
if (!policy) {
|
|
628
|
+
return null;
|
|
629
|
+
}
|
|
630
|
+
const cached = pleRangeRowCache.get(embedTokensPerLayer);
|
|
631
|
+
if (cached && cached.maxRows === policy.maxRows && cached.maxBytes === policy.maxBytes) {
|
|
632
|
+
return cached;
|
|
633
|
+
}
|
|
634
|
+
const next = {
|
|
635
|
+
maxRows: policy.maxRows,
|
|
636
|
+
maxBytes: policy.maxBytes,
|
|
637
|
+
totalBytes: 0,
|
|
638
|
+
rows: new Map(),
|
|
639
|
+
};
|
|
640
|
+
pleRangeRowCache.set(embedTokensPerLayer, next);
|
|
641
|
+
return next;
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
function touchPleCachedRow(cache, tokenId) {
|
|
645
|
+
const hit = cache?.rows?.get(tokenId) ?? null;
|
|
646
|
+
if (!hit) {
|
|
647
|
+
return null;
|
|
648
|
+
}
|
|
649
|
+
cache.rows.delete(tokenId);
|
|
650
|
+
cache.rows.set(tokenId, hit);
|
|
651
|
+
return hit.row;
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
function prunePleRangeRowCache(cache) {
|
|
655
|
+
while (cache.rows.size > cache.maxRows || cache.totalBytes > cache.maxBytes) {
|
|
656
|
+
const oldest = cache.rows.keys().next();
|
|
657
|
+
if (oldest.done) {
|
|
658
|
+
break;
|
|
659
|
+
}
|
|
660
|
+
const entry = cache.rows.get(oldest.value);
|
|
661
|
+
cache.rows.delete(oldest.value);
|
|
662
|
+
cache.totalBytes -= entry?.bytes ?? 0;
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
function cachePleRangeRow(cache, tokenId, row) {
|
|
667
|
+
if (!cache) {
|
|
668
|
+
return row;
|
|
669
|
+
}
|
|
670
|
+
const existing = cache.rows.get(tokenId);
|
|
671
|
+
if (existing) {
|
|
672
|
+
cache.totalBytes -= existing.bytes;
|
|
673
|
+
cache.rows.delete(tokenId);
|
|
674
|
+
}
|
|
675
|
+
cache.rows.set(tokenId, { row, bytes: row.byteLength });
|
|
676
|
+
cache.totalBytes += row.byteLength;
|
|
677
|
+
prunePleRangeRowCache(cache);
|
|
678
|
+
return row;
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
async function loadRangeBackedPleRow(
|
|
682
|
+
tokenId,
|
|
683
|
+
embedTokensPerLayer,
|
|
684
|
+
totalPerLayerHiddenSize,
|
|
685
|
+
sessionConfig,
|
|
686
|
+
label,
|
|
687
|
+
prefetchedRow = null
|
|
688
|
+
) {
|
|
689
|
+
if (!isCpuWeightBuffer(embedTokensPerLayer)) {
|
|
690
|
+
return null;
|
|
691
|
+
}
|
|
692
|
+
const cpuData = embedTokensPerLayer.data;
|
|
693
|
+
if (!isRangeBackedCpuEmbeddingSource(cpuData)) {
|
|
694
|
+
return null;
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
const cache = getPleRangeRowCache(embedTokensPerLayer, sessionConfig);
|
|
698
|
+
if (prefetchedRow && prefetchedRow.tokenId === tokenId) {
|
|
699
|
+
return cachePleRangeRow(cache, tokenId, prefetchedRow.row);
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
const cached = touchPleCachedRow(cache, tokenId);
|
|
703
|
+
if (cached) {
|
|
704
|
+
return cached;
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
const { sourceDtype, sourceRowBytes } = getPleRangeRowLoadConfig(
|
|
708
|
+
embedTokensPerLayer,
|
|
709
|
+
totalPerLayerHiddenSize
|
|
710
|
+
);
|
|
711
|
+
const chunk = normalizeRangeBytes(
|
|
712
|
+
await cpuData.loadRange(tokenId * sourceRowBytes, sourceRowBytes),
|
|
713
|
+
label
|
|
714
|
+
);
|
|
715
|
+
const row = new Float32Array(totalPerLayerHiddenSize);
|
|
716
|
+
decodeRangeChunkIntoOutput(chunk, sourceDtype, row, 0, totalPerLayerHiddenSize);
|
|
717
|
+
return cachePleRangeRow(cache, tokenId, row);
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
// Step 5: Prefetch next token's PLE row during current decode step.
|
|
721
|
+
// Returns a promise resolving to { tokenId, row: Float32Array } or null.
|
|
722
|
+
// Call after sampling produces the next token; pass result as options.prefetchedRow
|
|
723
|
+
// to the next preparePerLayerInputs call.
|
|
724
|
+
export function prefetchPerLayerRow(tokenId, embedTokensPerLayer, totalPerLayerHiddenSize, sessionConfig = null) {
|
|
725
|
+
if (!isCpuWeightBuffer(embedTokensPerLayer)) return null;
|
|
726
|
+
const cpuData = embedTokensPerLayer.data;
|
|
727
|
+
if (!isRangeBackedCpuEmbeddingSource(cpuData)) return null;
|
|
728
|
+
return loadRangeBackedPleRow(
|
|
729
|
+
tokenId,
|
|
730
|
+
embedTokensPerLayer,
|
|
731
|
+
totalPerLayerHiddenSize,
|
|
732
|
+
sessionConfig,
|
|
733
|
+
'Prefetched PLE row'
|
|
734
|
+
)
|
|
735
|
+
.then(row => (row ? { tokenId, row } : null))
|
|
736
|
+
.catch(() => null);
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
export function hasRangeBackedPerLayerInputEmbeddings(context) {
|
|
740
|
+
const hiddenSizePerLayerInput = Number(context?.config?.hiddenSizePerLayerInput ?? 0);
|
|
741
|
+
if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
|
|
742
|
+
return false;
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
const perLayerInputWeights = context?.weights?.get?.('per_layer_inputs');
|
|
746
|
+
if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
|
|
747
|
+
return false;
|
|
748
|
+
}
|
|
749
|
+
if (Array.isArray(perLayerInputWeights.embedTokensPerLayerSplit) && perLayerInputWeights.embedTokensPerLayerSplit.length > 0) {
|
|
750
|
+
return false;
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
const embedTokensPerLayer = perLayerInputWeights.embedTokensPerLayer;
|
|
754
|
+
return isCpuWeightBuffer(embedTokensPerLayer)
|
|
755
|
+
&& isRangeBackedCpuEmbeddingSource(embedTokensPerLayer.data);
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
export function hasGpuSplitPerLayerInputEmbeddings(context) {
|
|
759
|
+
const hiddenSizePerLayerInput = Number(context?.config?.hiddenSizePerLayerInput ?? 0);
|
|
760
|
+
if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
|
|
761
|
+
return false;
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
const perLayerInputWeights = context?.weights?.get?.('per_layer_inputs');
|
|
765
|
+
if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
|
|
766
|
+
return false;
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
return Array.isArray(perLayerInputWeights.embedTokensPerLayerSplit)
|
|
770
|
+
&& perLayerInputWeights.embedTokensPerLayerSplit.length > 0;
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
export async function ensurePleGpuSplitTablesRuntime(context) {
|
|
774
|
+
const policy = getPleSplitTablePolicy(context?.perLayerInputsSession ?? null);
|
|
775
|
+
if (!policy) {
|
|
776
|
+
return null;
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
const config = context?.config ?? null;
|
|
780
|
+
const hiddenSizePerLayerInput = Number(config?.hiddenSizePerLayerInput ?? 0);
|
|
781
|
+
const vocabSizePerLayerInput = Number(config?.vocabSizePerLayerInput ?? 0);
|
|
782
|
+
const numLayers = Number(config?.numLayers ?? 0);
|
|
783
|
+
if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
|
|
784
|
+
return null;
|
|
785
|
+
}
|
|
786
|
+
if (!Number.isFinite(vocabSizePerLayerInput) || vocabSizePerLayerInput <= 0) {
|
|
787
|
+
return null;
|
|
788
|
+
}
|
|
789
|
+
if (!Number.isFinite(numLayers) || numLayers <= 0) {
|
|
790
|
+
return null;
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
const perLayerInputWeights = getPerLayerInputWeights(context);
|
|
794
|
+
if (Array.isArray(perLayerInputWeights.embedTokensPerLayerSplit) && perLayerInputWeights.embedTokensPerLayerSplit.length === numLayers) {
|
|
795
|
+
return perLayerInputWeights.embedTokensPerLayerSplit;
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
const embedTokensPerLayer = perLayerInputWeights.embedTokensPerLayer;
|
|
799
|
+
if (!isCpuWeightBuffer(embedTokensPerLayer) || !isRangeBackedCpuEmbeddingSource(embedTokensPerLayer.data)) {
|
|
800
|
+
throw new Error('Gemma 4 gpu_split_tables materialization requires a range-backed CPU embedTokensPerLayer source.');
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
const sourceDtype = String(embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype ?? 'f32').toLowerCase();
|
|
804
|
+
if (sourceDtype !== 'f16' && sourceDtype !== 'f32') {
|
|
805
|
+
throw new Error(
|
|
806
|
+
`Gemma 4 gpu_split_tables materialization requires f16/f32 source rows; got "${sourceDtype}".`
|
|
807
|
+
);
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
const device = getDevice();
|
|
811
|
+
if (!device) {
|
|
812
|
+
throw new Error('No GPU device available for Gemma 4 gpu_split_tables materialization.');
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
const bytesPerElement = sourceDtype === 'f16' ? 2 : 4;
|
|
816
|
+
const totalPerLayerHiddenSize = numLayers * hiddenSizePerLayerInput;
|
|
817
|
+
const tableBytes = vocabSizePerLayerInput * hiddenSizePerLayerInput * bytesPerElement;
|
|
818
|
+
const splitTables = Array.from({ length: numLayers }, (_, layerIdx) => createWeightBuffer(
|
|
819
|
+
acquireBuffer(tableBytes, undefined, `L${layerIdx}.ple_table_split`),
|
|
820
|
+
sourceDtype,
|
|
821
|
+
'row',
|
|
822
|
+
[vocabSizePerLayerInput, hiddenSizePerLayerInput],
|
|
823
|
+
`L${layerIdx}.embed_tokens_per_layer_split`
|
|
824
|
+
));
|
|
825
|
+
|
|
826
|
+
try {
|
|
827
|
+
const rowsPerChunk = 128;
|
|
828
|
+
for (let rowStart = 0; rowStart < vocabSizePerLayerInput; rowStart += rowsPerChunk) {
|
|
829
|
+
const rowCount = Math.min(rowsPerChunk, vocabSizePerLayerInput - rowStart);
|
|
830
|
+
const chunkByteOffset = rowStart * totalPerLayerHiddenSize * bytesPerElement;
|
|
831
|
+
const chunkByteLength = rowCount * totalPerLayerHiddenSize * bytesPerElement;
|
|
832
|
+
const chunk = normalizeRangeBytes(
|
|
833
|
+
await embedTokensPerLayer.data.loadRange(chunkByteOffset, chunkByteLength),
|
|
834
|
+
'Gemma 4 split GPU PLE chunk'
|
|
835
|
+
);
|
|
836
|
+
|
|
837
|
+
if (sourceDtype === 'f16') {
|
|
838
|
+
const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, rowCount * totalPerLayerHiddenSize);
|
|
839
|
+
for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
|
|
840
|
+
const layerWords = new Uint16Array(rowCount * hiddenSizePerLayerInput);
|
|
841
|
+
for (let row = 0; row < rowCount; row += 1) {
|
|
842
|
+
const sourceStart = row * totalPerLayerHiddenSize + layerIdx * hiddenSizePerLayerInput;
|
|
843
|
+
layerWords.set(
|
|
844
|
+
sourceWords.subarray(sourceStart, sourceStart + hiddenSizePerLayerInput),
|
|
845
|
+
row * hiddenSizePerLayerInput
|
|
846
|
+
);
|
|
847
|
+
}
|
|
848
|
+
device.queue.writeBuffer(
|
|
849
|
+
splitTables[layerIdx].buffer,
|
|
850
|
+
rowStart * hiddenSizePerLayerInput * bytesPerElement,
|
|
851
|
+
layerWords.buffer,
|
|
852
|
+
layerWords.byteOffset,
|
|
853
|
+
layerWords.byteLength
|
|
854
|
+
);
|
|
855
|
+
}
|
|
856
|
+
} else {
|
|
857
|
+
const sourceValues = new Float32Array(chunk.buffer, chunk.byteOffset, rowCount * totalPerLayerHiddenSize);
|
|
858
|
+
for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
|
|
859
|
+
const layerValues = new Float32Array(rowCount * hiddenSizePerLayerInput);
|
|
860
|
+
for (let row = 0; row < rowCount; row += 1) {
|
|
861
|
+
const sourceStart = row * totalPerLayerHiddenSize + layerIdx * hiddenSizePerLayerInput;
|
|
862
|
+
layerValues.set(
|
|
863
|
+
sourceValues.subarray(sourceStart, sourceStart + hiddenSizePerLayerInput),
|
|
864
|
+
row * hiddenSizePerLayerInput
|
|
865
|
+
);
|
|
866
|
+
}
|
|
867
|
+
device.queue.writeBuffer(
|
|
868
|
+
splitTables[layerIdx].buffer,
|
|
869
|
+
rowStart * hiddenSizePerLayerInput * bytesPerElement,
|
|
870
|
+
layerValues.buffer,
|
|
871
|
+
layerValues.byteOffset,
|
|
872
|
+
layerValues.byteLength
|
|
873
|
+
);
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
} catch (error) {
|
|
878
|
+
for (const table of splitTables) {
|
|
879
|
+
releaseBuffer(table.buffer);
|
|
880
|
+
}
|
|
881
|
+
throw error;
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
perLayerInputWeights.embedTokensPerLayerSplit = splitTables;
|
|
885
|
+
return splitTables;
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
function resolvePleHotVocabularySeedTokenIds(context, maxTokens, vocabSizePerLayerInput) {
|
|
889
|
+
const rawSeedTokenIds = Array.isArray(context?.seedTokenIds) ? context.seedTokenIds : null;
|
|
890
|
+
if (!rawSeedTokenIds || rawSeedTokenIds.length === 0) {
|
|
891
|
+
return [];
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
const specialTokenIds = new Set();
|
|
895
|
+
const tokenizerSpecialTokens = context?.tokenizer?.getSpecialTokens?.() ?? null;
|
|
896
|
+
if (tokenizerSpecialTokens && typeof tokenizerSpecialTokens === 'object') {
|
|
897
|
+
for (const value of Object.values(tokenizerSpecialTokens)) {
|
|
898
|
+
if (Number.isInteger(value)) {
|
|
899
|
+
specialTokenIds.add(value);
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
const seeded = [];
|
|
905
|
+
const seen = new Set();
|
|
906
|
+
for (let i = rawSeedTokenIds.length - 1; i >= 0; i -= 1) {
|
|
907
|
+
const tokenId = rawSeedTokenIds[i];
|
|
908
|
+
if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSizePerLayerInput) {
|
|
909
|
+
continue;
|
|
910
|
+
}
|
|
911
|
+
if (specialTokenIds.has(tokenId) || seen.has(tokenId)) {
|
|
912
|
+
continue;
|
|
913
|
+
}
|
|
914
|
+
seeded.push(tokenId);
|
|
915
|
+
seen.add(tokenId);
|
|
916
|
+
if (seeded.length >= maxTokens) {
|
|
917
|
+
break;
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
return seeded;
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
function mergePleHotVocabularyTokenIds(seedTokenIds, tokenizerHotTokenIds, maxTokens, vocabSizePerLayerInput) {
|
|
924
|
+
const merged = [];
|
|
925
|
+
const seen = new Set();
|
|
926
|
+
|
|
927
|
+
for (const tokenId of seedTokenIds ?? []) {
|
|
928
|
+
if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSizePerLayerInput || seen.has(tokenId)) {
|
|
929
|
+
continue;
|
|
930
|
+
}
|
|
931
|
+
merged.push(tokenId);
|
|
932
|
+
seen.add(tokenId);
|
|
933
|
+
if (merged.length >= maxTokens) {
|
|
934
|
+
return merged;
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
for (const tokenId of tokenizerHotTokenIds ?? []) {
|
|
939
|
+
if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSizePerLayerInput || seen.has(tokenId)) {
|
|
940
|
+
continue;
|
|
941
|
+
}
|
|
942
|
+
merged.push(tokenId);
|
|
943
|
+
seen.add(tokenId);
|
|
944
|
+
if (merged.length >= maxTokens) {
|
|
945
|
+
break;
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
return merged;
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
953
|
+
const policy = getPleHotCachePolicy(context?.perLayerInputsSession ?? null);
|
|
954
|
+
if (!policy || policy.mode !== 'tokenizer_scores') {
|
|
955
|
+
return null;
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
const config = context?.config ?? null;
|
|
959
|
+
const hiddenSizePerLayerInput = Number(config?.hiddenSizePerLayerInput ?? 0);
|
|
960
|
+
const vocabSizePerLayerInput = Number(config?.vocabSizePerLayerInput ?? 0);
|
|
961
|
+
const numLayers = Number(config?.numLayers ?? 0);
|
|
962
|
+
if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
|
|
963
|
+
return null;
|
|
964
|
+
}
|
|
965
|
+
if (!Number.isFinite(vocabSizePerLayerInput) || vocabSizePerLayerInput <= 0) {
|
|
966
|
+
return null;
|
|
967
|
+
}
|
|
968
|
+
if (!Number.isFinite(numLayers) || numLayers <= 0) {
|
|
969
|
+
return null;
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
const tokenizer = context?.tokenizer ?? null;
|
|
973
|
+
const tokenizerHotTokenIds = typeof tokenizer?.getHotTokenIds === 'function'
|
|
974
|
+
? tokenizer.getHotTokenIds(policy.maxTokens)
|
|
975
|
+
: null;
|
|
976
|
+
const seedTokenIds = resolvePleHotVocabularySeedTokenIds(
|
|
977
|
+
context,
|
|
978
|
+
policy.maxTokens,
|
|
979
|
+
vocabSizePerLayerInput
|
|
980
|
+
);
|
|
981
|
+
const hotTokenIds = mergePleHotVocabularyTokenIds(
|
|
982
|
+
seedTokenIds,
|
|
983
|
+
tokenizerHotTokenIds,
|
|
984
|
+
policy.maxTokens,
|
|
985
|
+
vocabSizePerLayerInput
|
|
986
|
+
);
|
|
987
|
+
if (hotTokenIds.length === 0) {
|
|
988
|
+
return null;
|
|
989
|
+
}
|
|
990
|
+
const hotTokenIdsSignature = hotTokenIds.join(',');
|
|
991
|
+
|
|
992
|
+
const perLayerInputWeights = getPerLayerInputWeights(context);
|
|
993
|
+
const cached = perLayerInputWeights.embedTokensPerLayerHotRuntime ?? null;
|
|
994
|
+
if (
|
|
995
|
+
cached
|
|
996
|
+
&& cached.maxTokens === policy.maxTokens
|
|
997
|
+
&& cached.outputDtype === policy.outputDtype
|
|
998
|
+
&& cached.vocabSize === vocabSizePerLayerInput
|
|
999
|
+
&& cached.numLayers === numLayers
|
|
1000
|
+
&& cached.hotTokenIdsSignature === hotTokenIdsSignature
|
|
1001
|
+
) {
|
|
1002
|
+
return cached;
|
|
1003
|
+
}
|
|
1004
|
+
destroyPleHotVocabularyRuntime(perLayerInputWeights);
|
|
1005
|
+
|
|
1006
|
+
const embedTokensPerLayer = perLayerInputWeights.embedTokensPerLayer;
|
|
1007
|
+
if (!isCpuWeightBuffer(embedTokensPerLayer) || !isRangeBackedCpuEmbeddingSource(embedTokensPerLayer.data)) {
|
|
1008
|
+
return null;
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
const sourceDtype = String(embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype ?? 'f32').toLowerCase();
|
|
1012
|
+
if (sourceDtype !== policy.outputDtype) {
|
|
1013
|
+
throw new Error(
|
|
1014
|
+
`Gemma 4 hot vocabulary cache requires source dtype "${policy.outputDtype}" for zero-copy row packing; ` +
|
|
1015
|
+
`got "${sourceDtype}".`
|
|
1016
|
+
);
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
const device = getDevice();
|
|
1020
|
+
if (!device) {
|
|
1021
|
+
throw new Error('No GPU device available for Gemma 4 hot vocabulary cache.');
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
const bytesPerElement = policy.outputDtype === 'f16' ? 2 : 4;
|
|
1025
|
+
const totalPerLayerHiddenSize = numLayers * hiddenSizePerLayerInput;
|
|
1026
|
+
const sentinelIndex = hotTokenIds.length;
|
|
1027
|
+
const hotRowCount = sentinelIndex + 1;
|
|
1028
|
+
const splitTableBytes = hotRowCount * hiddenSizePerLayerInput * bytesPerElement;
|
|
1029
|
+
const splitTables = Array.from({ length: numLayers }, (_, layerIdx) => createWeightBuffer(
|
|
1030
|
+
acquireBuffer(splitTableBytes, undefined, `L${layerIdx}.ple_hot_vocab_table`),
|
|
1031
|
+
policy.outputDtype,
|
|
1032
|
+
'row',
|
|
1033
|
+
[hotRowCount, hiddenSizePerLayerInput],
|
|
1034
|
+
`L${layerIdx}.embed_tokens_per_layer_hot_vocab`
|
|
1035
|
+
));
|
|
1036
|
+
const hotTokenIndexMap = new Uint32Array(vocabSizePerLayerInput);
|
|
1037
|
+
hotTokenIndexMap.fill(sentinelIndex);
|
|
1038
|
+
for (let hotIndex = 0; hotIndex < hotTokenIds.length; hotIndex += 1) {
|
|
1039
|
+
const tokenId = hotTokenIds[hotIndex];
|
|
1040
|
+
if (Number.isInteger(tokenId) && tokenId >= 0 && tokenId < vocabSizePerLayerInput) {
|
|
1041
|
+
hotTokenIndexMap[tokenId] = hotIndex;
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
const hotTokenIndexMapBuffer = acquireBuffer(
|
|
1045
|
+
hotTokenIndexMap.byteLength,
|
|
1046
|
+
undefined,
|
|
1047
|
+
'ple_hot_token_index_map'
|
|
1048
|
+
);
|
|
1049
|
+
|
|
1050
|
+
try {
|
|
1051
|
+
device.queue.writeBuffer(hotTokenIndexMapBuffer, 0, hotTokenIndexMap);
|
|
1052
|
+
const zeroRow = policy.outputDtype === 'f16'
|
|
1053
|
+
? new Uint16Array(hiddenSizePerLayerInput)
|
|
1054
|
+
: new Float32Array(hiddenSizePerLayerInput);
|
|
1055
|
+
for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
|
|
1056
|
+
device.queue.writeBuffer(
|
|
1057
|
+
splitTables[layerIdx].buffer,
|
|
1058
|
+
sentinelIndex * hiddenSizePerLayerInput * bytesPerElement,
|
|
1059
|
+
zeroRow.buffer,
|
|
1060
|
+
zeroRow.byteOffset,
|
|
1061
|
+
zeroRow.byteLength
|
|
1062
|
+
);
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
const { sourceRowBytes } = getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize);
|
|
1066
|
+
for (let hotIndex = 0; hotIndex < hotTokenIds.length; hotIndex += 1) {
|
|
1067
|
+
const tokenId = hotTokenIds[hotIndex];
|
|
1068
|
+
const chunk = normalizeRangeBytes(
|
|
1069
|
+
await embedTokensPerLayer.data.loadRange(tokenId * sourceRowBytes, sourceRowBytes),
|
|
1070
|
+
'Gemma 4 hot vocabulary PLE row'
|
|
1071
|
+
);
|
|
1072
|
+
if (policy.outputDtype === 'f16') {
|
|
1073
|
+
const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
|
|
1074
|
+
for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
|
|
1075
|
+
const sourceStart = layerIdx * hiddenSizePerLayerInput;
|
|
1076
|
+
device.queue.writeBuffer(
|
|
1077
|
+
splitTables[layerIdx].buffer,
|
|
1078
|
+
hotIndex * hiddenSizePerLayerInput * bytesPerElement,
|
|
1079
|
+
sourceWords.buffer,
|
|
1080
|
+
sourceWords.byteOffset + sourceStart * bytesPerElement,
|
|
1081
|
+
hiddenSizePerLayerInput * bytesPerElement
|
|
1082
|
+
);
|
|
1083
|
+
}
|
|
1084
|
+
} else {
|
|
1085
|
+
const sourceValues = new Float32Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
|
|
1086
|
+
for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
|
|
1087
|
+
const sourceStart = layerIdx * hiddenSizePerLayerInput;
|
|
1088
|
+
device.queue.writeBuffer(
|
|
1089
|
+
splitTables[layerIdx].buffer,
|
|
1090
|
+
hotIndex * hiddenSizePerLayerInput * bytesPerElement,
|
|
1091
|
+
sourceValues.buffer,
|
|
1092
|
+
sourceValues.byteOffset + sourceStart * bytesPerElement,
|
|
1093
|
+
hiddenSizePerLayerInput * bytesPerElement
|
|
1094
|
+
);
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
} catch (error) {
|
|
1099
|
+
for (const table of splitTables) {
|
|
1100
|
+
releaseBuffer(table.buffer);
|
|
1101
|
+
}
|
|
1102
|
+
releaseBuffer(hotTokenIndexMapBuffer);
|
|
1103
|
+
throw error;
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
const runtime = {
|
|
1107
|
+
mode: 'tokenizer_scores',
|
|
1108
|
+
maxTokens: policy.maxTokens,
|
|
1109
|
+
outputDtype: policy.outputDtype,
|
|
1110
|
+
vocabSize: vocabSizePerLayerInput,
|
|
1111
|
+
numLayers,
|
|
1112
|
+
hotTokenIdsSignature,
|
|
1113
|
+
hotTokenIds: Uint32Array.from(hotTokenIds),
|
|
1114
|
+
hotTokenIndexMap,
|
|
1115
|
+
hotTokenIndexMapBuffer,
|
|
1116
|
+
sentinelIndex,
|
|
1117
|
+
splitTables,
|
|
1118
|
+
};
|
|
1119
|
+
perLayerInputWeights.embedTokensPerLayerHotRuntime = runtime;
|
|
1120
|
+
return runtime;
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context, options = {}) {
|
|
1124
|
+
const { config, weightConfig, debugFlags, recorder, decodeBuffers } = context;
|
|
1125
|
+
const hiddenSizePerLayerInput = Number(config.hiddenSizePerLayerInput ?? 0);
|
|
1126
|
+
if (!Number.isFinite(hiddenSizePerLayerInput) || hiddenSizePerLayerInput <= 0) {
|
|
1127
|
+
return null;
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
const vocabSizePerLayerInput = Number(config.vocabSizePerLayerInput ?? 0);
|
|
1131
|
+
if (!Number.isFinite(vocabSizePerLayerInput) || vocabSizePerLayerInput <= 0) {
|
|
1132
|
+
throw new Error(
|
|
1133
|
+
`Gemma 4 model "${config.modelId ?? 'unknown'}" requires architecture.vocabSizePerLayerInput ` +
|
|
1134
|
+
'when hiddenSizePerLayerInput is enabled.'
|
|
1135
|
+
);
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
const perLayerInputWeights = getPerLayerInputWeights(context);
|
|
1139
|
+
const embedTokensPerLayer = perLayerInputWeights.embedTokensPerLayer;
|
|
1140
|
+
const embedTokensPerLayerSplit = Array.isArray(perLayerInputWeights.embedTokensPerLayerSplit)
|
|
1141
|
+
? perLayerInputWeights.embedTokensPerLayerSplit
|
|
1142
|
+
: null;
|
|
1143
|
+
const hotVocabularyRuntime = getPleHotVocabularyRuntime(context);
|
|
1144
|
+
const perLayerModelProjection = perLayerInputWeights.perLayerModelProjection;
|
|
1145
|
+
const perLayerProjectionNorm = perLayerInputWeights.perLayerProjectionNorm;
|
|
1146
|
+
if (!embedTokensPerLayer || !perLayerModelProjection || !perLayerProjectionNorm) {
|
|
1147
|
+
throw new Error(
|
|
1148
|
+
'Gemma 4 per-layer inputs require embedTokensPerLayer, perLayerModelProjection, ' +
|
|
1149
|
+
'and perLayerProjectionNorm weights.'
|
|
1150
|
+
);
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
const numLayers = config.numLayers;
|
|
1154
|
+
const hasSplitEmbeddingTables = Array.isArray(embedTokensPerLayerSplit)
|
|
1155
|
+
&& embedTokensPerLayerSplit.length === numLayers;
|
|
1156
|
+
const hasHotVocabularyTables = Array.isArray(hotVocabularyRuntime?.splitTables)
|
|
1157
|
+
&& hotVocabularyRuntime.splitTables.length === numLayers;
|
|
1158
|
+
const numTokens = Number.isFinite(options.numTokens) ? options.numTokens : inputEmbedsTensor.shape?.[0];
|
|
1159
|
+
const indexOffset = Number.isFinite(options.indexOffset) ? options.indexOffset : 0;
|
|
1160
|
+
const perLayerIndexOffset = Number.isFinite(options.perLayerIndexOffset)
|
|
1161
|
+
? options.perLayerIndexOffset
|
|
1162
|
+
: indexOffset;
|
|
1163
|
+
if (!Number.isFinite(numTokens) || numTokens <= 0) {
|
|
1164
|
+
throw new Error('Gemma 4 per-layer inputs require a positive numTokens value.');
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
const activationDtype = selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', {
|
|
1168
|
+
dtype: inputEmbedsTensor.dtype,
|
|
1169
|
+
});
|
|
1170
|
+
const perLayerTokenIdsOption = options.perLayerTokenIds ?? null;
|
|
1171
|
+
let hotLocalTokenIds = null;
|
|
1172
|
+
if (
|
|
1173
|
+
!perLayerTokenIdsOption
|
|
1174
|
+
&& hasHotVocabularyTables
|
|
1175
|
+
&& numTokens === 1
|
|
1176
|
+
&& !isGpuBufferInstance(tokenIds)
|
|
1177
|
+
&& Array.isArray(tokenIds)
|
|
1178
|
+
&& Number.isInteger(tokenIds[0])
|
|
1179
|
+
) {
|
|
1180
|
+
const hotIndex = hotVocabularyRuntime.hotTokenIndexMap?.[tokenIds[0]] ?? hotVocabularyRuntime.sentinelIndex;
|
|
1181
|
+
if (hotIndex !== hotVocabularyRuntime.sentinelIndex) {
|
|
1182
|
+
hotLocalTokenIds = new Uint32Array([hotIndex]);
|
|
1183
|
+
if (context.stats) {
|
|
1184
|
+
context.stats.pleHotVocabularyHits = (context.stats.pleHotVocabularyHits ?? 0) + 1;
|
|
1185
|
+
}
|
|
1186
|
+
} else if (context.stats) {
|
|
1187
|
+
context.stats.pleHotVocabularyMisses = (context.stats.pleHotVocabularyMisses ?? 0) + 1;
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
const perLayerTokenIds = perLayerTokenIdsOption ?? hotLocalTokenIds;
|
|
1191
|
+
const useHotVocabularyTables = hasHotVocabularyTables && perLayerTokenIds != null;
|
|
1192
|
+
|
|
1193
|
+
const perLayerEmbeddingDtype = useHotVocabularyTables
|
|
1194
|
+
? hotVocabularyRuntime.outputDtype
|
|
1195
|
+
: hasSplitEmbeddingTables
|
|
1196
|
+
? getEmbeddingDtype(embedTokensPerLayerSplit[0])
|
|
1197
|
+
: getEmbeddingDtype(embedTokensPerLayer);
|
|
1198
|
+
const embedSource = hasSplitEmbeddingTables || useHotVocabularyTables
|
|
1199
|
+
? null
|
|
1200
|
+
: getEmbeddingSource(embedTokensPerLayer, 'embedTokensPerLayer');
|
|
1201
|
+
const totalPerLayerHiddenSize = numLayers * hiddenSizePerLayerInput;
|
|
1202
|
+
const hasRangeBackedProjection = isCpuWeightBuffer(perLayerModelProjection)
|
|
1203
|
+
&& isRangeBackedPleProjectionSource(perLayerModelProjection.data);
|
|
1204
|
+
const projectionWeight = hasRangeBackedProjection
|
|
1205
|
+
? null
|
|
1206
|
+
: getWeightBuffer(perLayerModelProjection, 'per_layer_model_projection');
|
|
1207
|
+
if (isWeightBuffer(perLayerModelProjection) && perLayerModelProjection.layout !== 'row') {
|
|
1208
|
+
throw new Error(
|
|
1209
|
+
'Gemma 4 per-layer input projection requires a row-major per_layer_model_projection weight. ' +
|
|
1210
|
+
`Got layout="${perLayerModelProjection.layout}".`
|
|
1211
|
+
);
|
|
1212
|
+
}
|
|
1213
|
+
const projectionScale = config.hiddenSize ** -0.5;
|
|
1214
|
+
const combineScale = 2 ** -0.5;
|
|
1215
|
+
const scaledProjectionNormWeight = await ensurePleScaledProjectionNormWeight(context, combineScale);
|
|
1216
|
+
const usesCachedScaledProjectionNormWeight = !!scaledProjectionNormWeight;
|
|
1217
|
+
const projectionNormWeight = scaledProjectionNormWeight ?? getNormWeightBuffer(
|
|
1218
|
+
perLayerProjectionNorm,
|
|
1219
|
+
'per_layer_projection_norm',
|
|
1220
|
+
weightConfig,
|
|
1221
|
+
debugFlags
|
|
1222
|
+
);
|
|
1223
|
+
const perLayerBuffers = new Array(numLayers).fill(null);
|
|
1224
|
+
const pleCache = options.pleCache ?? null;
|
|
1225
|
+
const activationBytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', {
|
|
1226
|
+
dtype: activationDtype,
|
|
1227
|
+
});
|
|
1228
|
+
const projectionDevice = getDevice();
|
|
1229
|
+
if (projectionDevice == null) {
|
|
1230
|
+
throw new Error('Gemma 4 per-layer input projection requires an initialized GPU device.');
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
// Decode-path optimizations: coalesced PLE read + fused projection matmul.
|
|
1234
|
+
// Gated on numTokens === 1 (decode) and row-major embeddings (non-transpose).
|
|
1235
|
+
// For numTokens > 1 (prefill), the fused matmul output is strided per-layer,
|
|
1236
|
+
// so we fall back to the per-layer path.
|
|
1237
|
+
const embedTranspose = (hasSplitEmbeddingTables || useHotVocabularyTables) ? false : getEmbeddingTranspose(embedTokensPerLayer);
|
|
1238
|
+
const canFuseDecodeOps = numTokens === 1 && !embedTranspose && !hasRangeBackedProjection;
|
|
1239
|
+
const tokenIdsAreGpuBuffer = isGpuBufferInstance(tokenIds);
|
|
1240
|
+
const decodeTokenId = canFuseDecodeOps && !tokenIdsAreGpuBuffer
|
|
1241
|
+
? Number(tokenIds[0])
|
|
1242
|
+
: null;
|
|
1243
|
+
|
|
1244
|
+
if (decodeTokenId != null) {
|
|
1245
|
+
const preparedTokenHit = getPreparedTokenEntry(
|
|
1246
|
+
pleCache,
|
|
1247
|
+
decodeTokenId,
|
|
1248
|
+
context.perLayerInputsSession ?? null,
|
|
1249
|
+
activationDtype,
|
|
1250
|
+
context.stats ?? null
|
|
1251
|
+
);
|
|
1252
|
+
if (preparedTokenHit) {
|
|
1253
|
+
return preparedTokenHit;
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
// Step 5: Use prefetched PLE row if available and token matches.
|
|
1258
|
+
// Falls back to inline coalesced read (step 3) otherwise.
|
|
1259
|
+
let preloadedCpuRow = null;
|
|
1260
|
+
if (canFuseDecodeOps && !perLayerTokenIds) {
|
|
1261
|
+
const embedCpuData = !hasSplitEmbeddingTables && isCpuWeightBuffer(embedSource) ? embedSource.data : null;
|
|
1262
|
+
if (embedCpuData && isRangeBackedCpuEmbeddingSource(embedCpuData)) {
|
|
1263
|
+
if (tokenIdsAreGpuBuffer) {
|
|
1264
|
+
throw new Error(
|
|
1265
|
+
'Gemma 4 per-layer input decode with range-backed CPU embeddings requires CPU token IDs. ' +
|
|
1266
|
+
'Disable batch decode or use GPU-resident per-layer inputs.'
|
|
1267
|
+
);
|
|
1268
|
+
}
|
|
1269
|
+
preloadedCpuRow = await loadRangeBackedPleRow(
|
|
1270
|
+
tokenIds[0],
|
|
1271
|
+
embedTokensPerLayer,
|
|
1272
|
+
totalPerLayerHiddenSize,
|
|
1273
|
+
context.perLayerInputsSession ?? null,
|
|
1274
|
+
'Coalesced PLE row',
|
|
1275
|
+
options.prefetchedRow ?? null
|
|
1276
|
+
);
|
|
1277
|
+
if (!preloadedCpuRow) {
|
|
1278
|
+
throw new Error('Gemma 4 range-backed per-layer input row load returned null unexpectedly.');
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1283
|
+
// Step 6: Batched prefill gather. When numTokens > 1 and the PLE source is
|
|
1284
|
+
// range-backed + row-major, read all tokens' full PLE rows into a single CPU
|
|
1285
|
+
// buffer. This avoids numTokens × numLayers separate loadRange calls during
|
|
1286
|
+
// the per-layer embed loop.
|
|
1287
|
+
let prefillBatchedRows = null;
|
|
1288
|
+
if (!canFuseDecodeOps && numTokens > 1 && !perLayerTokenIds && !getEmbeddingTranspose(embedTokensPerLayer)) {
|
|
1289
|
+
const embedCpuData = !hasSplitEmbeddingTables && isCpuWeightBuffer(embedSource) ? embedSource.data : null;
|
|
1290
|
+
if (embedCpuData && isRangeBackedCpuEmbeddingSource(embedCpuData)) {
|
|
1291
|
+
const tokenIdArray = Array.isArray(tokenIds) ? tokenIds : Array.from(tokenIds);
|
|
1292
|
+
prefillBatchedRows = new Float32Array(numTokens * totalPerLayerHiddenSize);
|
|
1293
|
+
for (let t = 0; t < numTokens; t++) {
|
|
1294
|
+
const row = await loadRangeBackedPleRow(
|
|
1295
|
+
tokenIdArray[t],
|
|
1296
|
+
embedTokensPerLayer,
|
|
1297
|
+
totalPerLayerHiddenSize,
|
|
1298
|
+
context.perLayerInputsSession ?? null,
|
|
1299
|
+
'Batched PLE prefill row'
|
|
1300
|
+
);
|
|
1301
|
+
if (!row) {
|
|
1302
|
+
throw new Error('Gemma 4 batched per-layer input row load returned null unexpectedly.');
|
|
1303
|
+
}
|
|
1304
|
+
prefillBatchedRows.set(row, t * totalPerLayerHiddenSize);
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
// Fused projection matmul: one dispatch for all layers instead of numLayers dispatches.
|
|
1310
|
+
// Produces [numTokens × totalPerLayerHiddenSize], then scales the full output and
|
|
1311
|
+
// extracts per-layer slices via GPU buffer copies.
|
|
1312
|
+
let fusedProjectionSlices = null;
|
|
1313
|
+
if (canFuseDecodeOps) {
|
|
1314
|
+
const fusedProjection = await doMatmul(
|
|
1315
|
+
inputEmbedsTensor,
|
|
1316
|
+
projectionWeight,
|
|
1317
|
+
numTokens,
|
|
1318
|
+
totalPerLayerHiddenSize,
|
|
1319
|
+
config.hiddenSize,
|
|
1320
|
+
{
|
|
1321
|
+
transposeB: 'auto',
|
|
1322
|
+
label: 'per_layer_fused_projection',
|
|
1323
|
+
kernelPath: context.kernelPath ?? null,
|
|
1324
|
+
role: 'per_layer_model_projection',
|
|
1325
|
+
outputDtype: activationDtype,
|
|
1326
|
+
},
|
|
1327
|
+
recorder
|
|
1328
|
+
);
|
|
1329
|
+
|
|
1330
|
+
// One scale dispatch instead of numLayers
|
|
1331
|
+
const scaledFused = recorder
|
|
1332
|
+
? await recordScale(recorder, fusedProjection, projectionScale, {
|
|
1333
|
+
count: numTokens * totalPerLayerHiddenSize,
|
|
1334
|
+
})
|
|
1335
|
+
: await runScale(fusedProjection, projectionScale, {
|
|
1336
|
+
count: numTokens * totalPerLayerHiddenSize,
|
|
1337
|
+
});
|
|
1338
|
+
releaseOrTrack(recorder, fusedProjection.buffer, decodeBuffers);
|
|
1339
|
+
|
|
1340
|
+
// Step 4: Extract per-layer slices via GPU buffer copies (one encoder, one submit).
|
|
1341
|
+
// Reuse cached slice buffers when available to avoid per-step pool churn.
|
|
1342
|
+
const device = getDevice();
|
|
1343
|
+
const sliceBytes = hiddenSizePerLayerInput * activationBytesPerElement;
|
|
1344
|
+
const encoder = recorder ? recorder.getEncoder() : device.createCommandEncoder();
|
|
1345
|
+
fusedProjectionSlices = new Array(numLayers);
|
|
1346
|
+
for (let l = 0; l < numLayers; l++) {
|
|
1347
|
+
const sliceBuf = pleCache?.sliceBuffers?.[l] ?? acquireBuffer(sliceBytes, undefined, `L${l}.per_layer_proj_slice`);
|
|
1348
|
+
encoder.copyBufferToBuffer(scaledFused.buffer, l * sliceBytes, sliceBuf, 0, sliceBytes);
|
|
1349
|
+
fusedProjectionSlices[l] = sliceBuf;
|
|
1350
|
+
}
|
|
1351
|
+
if (!recorder) {
|
|
1352
|
+
device.queue.submit([encoder.finish()]);
|
|
1353
|
+
}
|
|
1354
|
+
releaseOrTrack(recorder, scaledFused.buffer, decodeBuffers);
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
const embedDtypeResolved = selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', {
|
|
1358
|
+
dtype: perLayerEmbeddingDtype,
|
|
1359
|
+
});
|
|
1360
|
+
|
|
1361
|
+
try {
|
|
1362
|
+
for (let layerIdx = 0; layerIdx < numLayers; layerIdx++) {
|
|
1363
|
+
const hiddenOffset = layerIdx * hiddenSizePerLayerInput;
|
|
1364
|
+
const layerEmbedSource = useHotVocabularyTables
|
|
1365
|
+
? getEmbeddingSource(hotVocabularyRuntime.splitTables[layerIdx], `embedTokensPerLayerHot[L${layerIdx}]`)
|
|
1366
|
+
: hasSplitEmbeddingTables
|
|
1367
|
+
? getEmbeddingSource(embedTokensPerLayerSplit[layerIdx], `embedTokensPerLayerSplit[L${layerIdx}]`)
|
|
1368
|
+
: embedSource;
|
|
1369
|
+
let gatheredTensor = null;
|
|
1370
|
+
let scaledProjectionTensor = null;
|
|
1371
|
+
let combinedTensor = null;
|
|
1372
|
+
try {
|
|
1373
|
+
gatheredTensor = await embed(perLayerTokenIds ?? tokenIds, layerEmbedSource, {
|
|
1374
|
+
hiddenSize: hiddenSizePerLayerInput,
|
|
1375
|
+
vocabSize: useHotVocabularyTables ? (hotVocabularyRuntime.sentinelIndex + 1) : vocabSizePerLayerInput,
|
|
1376
|
+
scaleEmbeddings: true,
|
|
1377
|
+
recorder,
|
|
1378
|
+
numTokens,
|
|
1379
|
+
indexOffset: perLayerTokenIds ? perLayerIndexOffset : indexOffset,
|
|
1380
|
+
transpose: embedTranspose,
|
|
1381
|
+
debugProbes: context.debugProbes,
|
|
1382
|
+
operatorDiagnostics: context.operatorDiagnostics,
|
|
1383
|
+
activationDtype,
|
|
1384
|
+
embeddingDtype: embedDtypeResolved,
|
|
1385
|
+
executionPolicies: context.executionPolicies ?? null,
|
|
1386
|
+
inputHiddenSize: (hasSplitEmbeddingTables || useHotVocabularyTables) ? hiddenSizePerLayerInput : totalPerLayerHiddenSize,
|
|
1387
|
+
hiddenOffset: (hasSplitEmbeddingTables || useHotVocabularyTables) ? 0 : hiddenOffset,
|
|
1388
|
+
preloadedCpuRow,
|
|
1389
|
+
preloadedCpuBatchedRows: prefillBatchedRows,
|
|
1390
|
+
outputBuffer: canFuseDecodeOps
|
|
1391
|
+
? (pleCache?.gatherSliceBuffers?.[layerIdx] ?? undefined)
|
|
1392
|
+
: undefined,
|
|
1393
|
+
});
|
|
1394
|
+
|
|
1395
|
+
if (fusedProjectionSlices) {
|
|
1396
|
+
// Use pre-computed fused projection slice (already scaled).
|
|
1397
|
+
scaledProjectionTensor = createTensor(
|
|
1398
|
+
fusedProjectionSlices[layerIdx],
|
|
1399
|
+
activationDtype,
|
|
1400
|
+
[numTokens, hiddenSizePerLayerInput],
|
|
1401
|
+
`L${layerIdx}.per_layer_proj_scaled`
|
|
1402
|
+
);
|
|
1403
|
+
fusedProjectionSlices[layerIdx] = null;
|
|
1404
|
+
} else {
|
|
1405
|
+
let projectionWeightForLayer = projectionWeight;
|
|
1406
|
+
let projectionWeightBufferForLayer = null;
|
|
1407
|
+
let projectionWeightOffset = 0;
|
|
1408
|
+
let projectedTensor = null;
|
|
1409
|
+
try {
|
|
1410
|
+
const rangeBackedProjection = await loadRangeBackedPleProjectionSliceBytes(
|
|
1411
|
+
perLayerModelProjection,
|
|
1412
|
+
layerIdx,
|
|
1413
|
+
hiddenSizePerLayerInput,
|
|
1414
|
+
config.hiddenSize,
|
|
1415
|
+
`L${layerIdx}.per_layer_projection_in`
|
|
1416
|
+
);
|
|
1417
|
+
if (rangeBackedProjection) {
|
|
1418
|
+
projectionWeightBufferForLayer = acquireBuffer(
|
|
1419
|
+
rangeBackedProjection.bytes.byteLength,
|
|
1420
|
+
undefined,
|
|
1421
|
+
`L${layerIdx}.per_layer_projection_in_weight`
|
|
1422
|
+
);
|
|
1423
|
+
projectionDevice.queue.writeBuffer(
|
|
1424
|
+
projectionWeightBufferForLayer,
|
|
1425
|
+
0,
|
|
1426
|
+
rangeBackedProjection.bytes,
|
|
1427
|
+
rangeBackedProjection.bytes.byteOffset,
|
|
1428
|
+
rangeBackedProjection.bytes.byteLength
|
|
1429
|
+
);
|
|
1430
|
+
projectionWeightForLayer = createWeightBuffer(
|
|
1431
|
+
projectionWeightBufferForLayer,
|
|
1432
|
+
rangeBackedProjection.dtype,
|
|
1433
|
+
rangeBackedProjection.layout,
|
|
1434
|
+
rangeBackedProjection.shape,
|
|
1435
|
+
`L${layerIdx}.per_layer_projection_in_weight`
|
|
1436
|
+
);
|
|
1437
|
+
projectionWeightOffset = 0;
|
|
1438
|
+
} else {
|
|
1439
|
+
projectionWeightForLayer = resolveDensePleProjectionWeight(
|
|
1440
|
+
projectionWeightForLayer,
|
|
1441
|
+
`L${layerIdx}.per_layer_projection_in_weight`
|
|
1442
|
+
);
|
|
1443
|
+
const projectionWeightDtype = selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', {
|
|
1444
|
+
dtype: getPleProjectionWeightDtype(projectionWeightForLayer),
|
|
1445
|
+
});
|
|
1446
|
+
const projectionWeightBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', {
|
|
1447
|
+
dtype: projectionWeightDtype,
|
|
1448
|
+
});
|
|
1449
|
+
projectionWeightOffset = hiddenOffset * config.hiddenSize * projectionWeightBytes;
|
|
1450
|
+
}
|
|
1451
|
+
projectedTensor = await doMatmul(
|
|
1452
|
+
inputEmbedsTensor,
|
|
1453
|
+
projectionWeightForLayer,
|
|
1454
|
+
numTokens,
|
|
1455
|
+
hiddenSizePerLayerInput,
|
|
1456
|
+
config.hiddenSize,
|
|
1457
|
+
{
|
|
1458
|
+
transposeB: 'auto',
|
|
1459
|
+
bOffset: projectionWeightOffset,
|
|
1460
|
+
label: `L${layerIdx}.per_layer_projection_in`,
|
|
1461
|
+
layerIdx,
|
|
1462
|
+
kernelPath: context.kernelPath ?? null,
|
|
1463
|
+
role: 'per_layer_model_projection',
|
|
1464
|
+
outputDtype: activationDtype,
|
|
1465
|
+
},
|
|
1466
|
+
recorder
|
|
1467
|
+
);
|
|
1468
|
+
} finally {
|
|
1469
|
+
if (projectionWeightBufferForLayer) {
|
|
1470
|
+
releaseOrTrack(recorder, projectionWeightBufferForLayer, decodeBuffers);
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
scaledProjectionTensor = recorder
|
|
1474
|
+
? await recordScale(recorder, projectedTensor, projectionScale, {
|
|
1475
|
+
count: numTokens * hiddenSizePerLayerInput,
|
|
1476
|
+
})
|
|
1477
|
+
: await runScale(projectedTensor, projectionScale, {
|
|
1478
|
+
count: numTokens * hiddenSizePerLayerInput,
|
|
1479
|
+
});
|
|
1480
|
+
releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
|
|
1481
|
+
projectedTensor = null;
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1484
|
+
// Fuse the residual add into RMSNorm so Gemma 4 PLE decode avoids an
|
|
1485
|
+
// extra dispatch per layer. When the model uses raw RMSNorm weights,
|
|
1486
|
+
// cache the fixed combine scale into the norm weights and skip the
|
|
1487
|
+
// legacy post-norm scale dispatch entirely.
|
|
1488
|
+
combinedTensor = await doRMSNorm(scaledProjectionTensor, projectionNormWeight, config.rmsNormEps, {
|
|
1489
|
+
batchSize: numTokens,
|
|
1490
|
+
hiddenSize: hiddenSizePerLayerInput,
|
|
1491
|
+
residual: gatheredTensor,
|
|
1492
|
+
label: `L${layerIdx}.per_layer_input_combine`,
|
|
1493
|
+
layerIdx,
|
|
1494
|
+
rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
|
|
1495
|
+
}, recorder);
|
|
1496
|
+
releasePleSliceBuffer(recorder, scaledProjectionTensor.buffer, decodeBuffers, pleCache);
|
|
1497
|
+
scaledProjectionTensor = null;
|
|
1498
|
+
releasePleSliceBuffer(recorder, gatheredTensor.buffer, decodeBuffers, pleCache);
|
|
1499
|
+
gatheredTensor = null;
|
|
1500
|
+
|
|
1501
|
+
if (usesCachedScaledProjectionNormWeight) {
|
|
1502
|
+
perLayerBuffers[layerIdx] = combinedTensor.buffer;
|
|
1503
|
+
combinedTensor = null;
|
|
1504
|
+
} else {
|
|
1505
|
+
// Step 8: Inplace scale avoids allocating a separate output buffer.
|
|
1506
|
+
// The combined tensor buffer is reused as the final per-layer output.
|
|
1507
|
+
const scaledTensor = recorder
|
|
1508
|
+
? await recordScale(recorder, combinedTensor, combineScale, {
|
|
1509
|
+
count: numTokens * hiddenSizePerLayerInput,
|
|
1510
|
+
inplace: true,
|
|
1511
|
+
})
|
|
1512
|
+
: await runScale(combinedTensor, combineScale, {
|
|
1513
|
+
count: numTokens * hiddenSizePerLayerInput,
|
|
1514
|
+
inplace: true,
|
|
1515
|
+
});
|
|
1516
|
+
perLayerBuffers[layerIdx] = scaledTensor.buffer;
|
|
1517
|
+
combinedTensor = null;
|
|
1518
|
+
}
|
|
1519
|
+
await runProbes('per_layer_input', perLayerBuffers[layerIdx], {
|
|
1520
|
+
layerIdx,
|
|
1521
|
+
numTokens,
|
|
1522
|
+
hiddenSize: hiddenSizePerLayerInput,
|
|
1523
|
+
probes: context.debugProbes,
|
|
1524
|
+
recorder,
|
|
1525
|
+
operatorDiagnostics: context.operatorDiagnostics,
|
|
1526
|
+
dtype: activationDtype,
|
|
1527
|
+
});
|
|
1528
|
+
} catch (error) {
|
|
1529
|
+
if (combinedTensor) {
|
|
1530
|
+
releaseOrTrack(recorder, combinedTensor.buffer, decodeBuffers);
|
|
1531
|
+
}
|
|
1532
|
+
if (gatheredTensor) {
|
|
1533
|
+
releasePleSliceBuffer(recorder, gatheredTensor.buffer, decodeBuffers, pleCache);
|
|
1534
|
+
}
|
|
1535
|
+
if (scaledProjectionTensor) {
|
|
1536
|
+
releasePleSliceBuffer(recorder, scaledProjectionTensor.buffer, decodeBuffers, pleCache);
|
|
1537
|
+
}
|
|
1538
|
+
throw error;
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
} catch (error) {
|
|
1542
|
+
if (fusedProjectionSlices) {
|
|
1543
|
+
for (let i = 0; i < fusedProjectionSlices.length; i++) {
|
|
1544
|
+
const buf = fusedProjectionSlices[i];
|
|
1545
|
+
releasePleSliceBuffer(recorder, buf, decodeBuffers, pleCache);
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
for (const buffer of perLayerBuffers) {
|
|
1549
|
+
if (buffer) {
|
|
1550
|
+
releaseOrTrack(recorder, buffer, decodeBuffers);
|
|
1551
|
+
}
|
|
1552
|
+
}
|
|
1553
|
+
throw error;
|
|
1554
|
+
} finally {
|
|
1555
|
+
if (!usesCachedScaledProjectionNormWeight && !isGpuBufferInstance(perLayerProjectionNorm)) {
|
|
1556
|
+
releaseOrTrack(recorder, projectionNormWeight, decodeBuffers);
|
|
1557
|
+
}
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
if (decodeTokenId != null) {
|
|
1561
|
+
return storePreparedTokenEntry(
|
|
1562
|
+
pleCache,
|
|
1563
|
+
decodeTokenId,
|
|
1564
|
+
perLayerBuffers,
|
|
1565
|
+
context.perLayerInputsSession ?? null,
|
|
1566
|
+
activationDtype,
|
|
1567
|
+
context.stats ?? null
|
|
1568
|
+
);
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
return perLayerBuffers;
|
|
1572
|
+
}
|
|
1573
|
+
|
|
1574
|
+
export function createPerLayerInputTensor(buffer, numTokens, hiddenSizePerLayerInput, activationDtype) {
|
|
1575
|
+
return createTensor(
|
|
1576
|
+
buffer,
|
|
1577
|
+
activationDtype,
|
|
1578
|
+
[numTokens, hiddenSizePerLayerInput],
|
|
1579
|
+
'per_layer_input'
|
|
1580
|
+
);
|
|
1581
|
+
}
|