doppler-gpu 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
|
|
2
2
|
import { getDevice, initDevice, getKernelCapabilities } from '../../gpu/device.js';
|
|
3
|
-
import { getBufferPool as getGlobalBufferPool } from '../../memory/buffer-pool.js';
|
|
3
|
+
import { getBufferPool as getGlobalBufferPool, releaseBuffer } from '../../memory/buffer-pool.js';
|
|
4
4
|
import { log } from '../../debug/index.js';
|
|
5
5
|
import { configurePerfGuards } from '../../gpu/perf-guards.js';
|
|
6
6
|
import { MoERouter } from '../moe-router.js';
|
|
@@ -23,6 +23,7 @@ import {
|
|
|
23
23
|
initEmulation,
|
|
24
24
|
destroyEmulation,
|
|
25
25
|
} from './text/init.js';
|
|
26
|
+
import { formatChatMessages } from './text/chat-format.js';
|
|
26
27
|
import {
|
|
27
28
|
runKernelWarmup,
|
|
28
29
|
applyModelBatchingRuntimeDefaults,
|
|
@@ -46,6 +47,7 @@ import { getDopplerLoader } from '../../loader/doppler-loader.js';
|
|
|
46
47
|
import { registerPipeline, getPipelineFactory } from './registry.js';
|
|
47
48
|
import { selectRuleValue } from '../../rules/rule-registry.js';
|
|
48
49
|
import { initConvLayerState } from './text/ops.js';
|
|
50
|
+
import { destroyPleBufferCache, destroyPleRuntimeCache } from './text/per-layer-inputs.js';
|
|
49
51
|
|
|
50
52
|
function destroyMoERouter(router) {
|
|
51
53
|
if (router && typeof router.destroy === 'function') {
|
|
@@ -53,6 +55,93 @@ function destroyMoERouter(router) {
|
|
|
53
55
|
}
|
|
54
56
|
}
|
|
55
57
|
|
|
58
|
+
function resolveSingleSpecialTokenId(tokenizer, tokenText, label) {
|
|
59
|
+
const rawTokenIds = tokenizer?.encode?.(tokenText);
|
|
60
|
+
const tokenIds = Array.isArray(rawTokenIds)
|
|
61
|
+
? rawTokenIds
|
|
62
|
+
: (ArrayBuffer.isView(rawTokenIds) ? Array.from(rawTokenIds) : null);
|
|
63
|
+
if (!Array.isArray(tokenIds) || tokenIds.length !== 1) {
|
|
64
|
+
throw new Error(
|
|
65
|
+
`[Pipeline] transcribeImage: tokenizer must encode ${label} "${tokenText}" as exactly one token.`
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
const tokenId = Number(tokenIds[0]);
|
|
69
|
+
if (!Number.isFinite(tokenId) || Math.floor(tokenId) !== tokenId || tokenId < 0) {
|
|
70
|
+
throw new Error(
|
|
71
|
+
`[Pipeline] transcribeImage: tokenizer returned invalid ${label} token id "${tokenIds[0]}".`
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
return tokenId;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function expandImagePlaceholderTokenIds(tokenIds, imageTokenId, numImageTokens, options = {}) {
|
|
78
|
+
const normalizedTokenIds = Array.isArray(tokenIds)
|
|
79
|
+
? Int32Array.from(tokenIds)
|
|
80
|
+
: (ArrayBuffer.isView(tokenIds) ? Int32Array.from(tokenIds) : null);
|
|
81
|
+
if (!(normalizedTokenIds instanceof Int32Array)) {
|
|
82
|
+
throw new Error(
|
|
83
|
+
'[Pipeline] transcribeImage: tokenizer.encode() must return an array or typed array of token IDs.'
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
if (!Number.isFinite(numImageTokens) || Math.floor(numImageTokens) !== numImageTokens || numImageTokens < 1) {
|
|
87
|
+
throw new Error(
|
|
88
|
+
`[Pipeline] transcribeImage: image token span must be a positive integer, got ${numImageTokens}.`
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
let placeholderIndex = -1;
|
|
93
|
+
let placeholderCount = 0;
|
|
94
|
+
for (let i = 0; i < normalizedTokenIds.length; i++) {
|
|
95
|
+
if (normalizedTokenIds[i] !== imageTokenId) continue;
|
|
96
|
+
if (placeholderIndex < 0) {
|
|
97
|
+
placeholderIndex = i;
|
|
98
|
+
}
|
|
99
|
+
placeholderCount++;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
if (placeholderCount !== 1) {
|
|
103
|
+
throw new Error(
|
|
104
|
+
`[Pipeline] transcribeImage: expected exactly one image_token_id (${imageTokenId}) placeholder ` +
|
|
105
|
+
`from the chat template, got ${placeholderCount}.`
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const boiTokenId = Number.isInteger(options.boiTokenId) ? options.boiTokenId : null;
|
|
110
|
+
const eoiTokenId = Number.isInteger(options.eoiTokenId) ? options.eoiTokenId : null;
|
|
111
|
+
const prefixExtra = boiTokenId == null ? 0 : 1;
|
|
112
|
+
const suffixExtra = eoiTokenId == null ? 0 : 1;
|
|
113
|
+
const expandedLength = normalizedTokenIds.length - 1 + prefixExtra + numImageTokens + suffixExtra;
|
|
114
|
+
const expanded = new Int32Array(expandedLength);
|
|
115
|
+
expanded.set(normalizedTokenIds.subarray(0, placeholderIndex), 0);
|
|
116
|
+
let writeOffset = placeholderIndex;
|
|
117
|
+
if (boiTokenId != null) {
|
|
118
|
+
expanded[writeOffset++] = boiTokenId;
|
|
119
|
+
}
|
|
120
|
+
expanded.fill(imageTokenId, writeOffset, writeOffset + numImageTokens);
|
|
121
|
+
const imageStartOffset = writeOffset;
|
|
122
|
+
writeOffset += numImageTokens;
|
|
123
|
+
if (eoiTokenId != null) {
|
|
124
|
+
expanded[writeOffset++] = eoiTokenId;
|
|
125
|
+
}
|
|
126
|
+
expanded.set(
|
|
127
|
+
normalizedTokenIds.subarray(placeholderIndex + 1),
|
|
128
|
+
writeOffset
|
|
129
|
+
);
|
|
130
|
+
|
|
131
|
+
return {
|
|
132
|
+
inputIds: expanded,
|
|
133
|
+
imageStartOffset,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export function buildConservativeMultimodalGenerationOptions(options = {}) {
|
|
138
|
+
return {
|
|
139
|
+
...options,
|
|
140
|
+
disableCommandBatching: true,
|
|
141
|
+
disableMultiTokenDecode: true,
|
|
142
|
+
stopCheckMode: 'per-token',
|
|
143
|
+
};
|
|
144
|
+
}
|
|
56
145
|
|
|
57
146
|
// ============================================================================
|
|
58
147
|
// Main Inference Pipeline Class
|
|
@@ -68,6 +157,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
68
157
|
|
|
69
158
|
|
|
70
159
|
_preloadedWeights = null;
|
|
160
|
+
runtimeOverrides = null;
|
|
71
161
|
|
|
72
162
|
constructor() {
|
|
73
163
|
super();
|
|
@@ -90,6 +180,11 @@ export class InferencePipeline extends PipelineState {
|
|
|
90
180
|
assignStorageContext: true,
|
|
91
181
|
});
|
|
92
182
|
this.runtimeConfig = runtimeConfig;
|
|
183
|
+
this.runtimeOverrides = contexts.runtimeConfig == null
|
|
184
|
+
? null
|
|
185
|
+
: (typeof structuredClone === 'function'
|
|
186
|
+
? structuredClone(contexts.runtimeConfig)
|
|
187
|
+
: JSON.parse(JSON.stringify(contexts.runtimeConfig)));
|
|
93
188
|
applyPipelineDebugConfig(sharedDebug?.pipeline);
|
|
94
189
|
configurePerfGuards(sharedDebug?.perfGuards);
|
|
95
190
|
|
|
@@ -116,6 +211,10 @@ export class InferencePipeline extends PipelineState {
|
|
|
116
211
|
const loadStart = performance.now();
|
|
117
212
|
this.manifest = manifest;
|
|
118
213
|
this.decodeRing?.release();
|
|
214
|
+
if (this.sampleReadbackBuffer) {
|
|
215
|
+
this.sampleReadbackBuffer.destroy();
|
|
216
|
+
this.sampleReadbackBuffer = null;
|
|
217
|
+
}
|
|
119
218
|
this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
|
|
120
219
|
destroyMoERouter(this.moeRouter);
|
|
121
220
|
this.moeRouter = null;
|
|
@@ -131,7 +230,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
131
230
|
// Phase 1 — applyExecutionV1RuntimeConfig
|
|
132
231
|
// Reads: manifest.inference.execution, kernelCapabilities, platform
|
|
133
232
|
// Mutates: runtimeConfig.inference (kernelPath, pipeline, compute,
|
|
134
|
-
//
|
|
233
|
+
// session via runtimeInferencePatch)
|
|
135
234
|
//
|
|
136
235
|
// Phase 2 — parseModelConfig + applyModelBatchingRuntimeDefaults
|
|
137
236
|
// Reads: manifest.architecture, runtimeConfig.inference.modelOverrides
|
|
@@ -141,7 +240,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
141
240
|
// Phase 3 — resolveKernelPathState
|
|
142
241
|
// Reads: manifest, modelConfig.kernelPath, runtimeConfig.inference.kernelPath
|
|
143
242
|
// Mutates: runtimeConfig.inference.compute.activationDtype,
|
|
144
|
-
// runtimeConfig.inference.kvcache.kvDtype,
|
|
243
|
+
// runtimeConfig.inference.session.kvcache.kvDtype,
|
|
145
244
|
// runtimeConfig.inference.session.compute.defaults.outputDtype
|
|
146
245
|
//
|
|
147
246
|
// Phase 4 — _resolveLayerPipeline
|
|
@@ -171,6 +270,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
171
270
|
|
|
172
271
|
const executionV1Runtime = applyExecutionV1RuntimeConfig({
|
|
173
272
|
runtimeConfig: this.runtimeConfig,
|
|
273
|
+
runtimeOverrides: this.runtimeOverrides,
|
|
174
274
|
manifest,
|
|
175
275
|
modelId: manifest.modelId ?? 'model',
|
|
176
276
|
numLayers: Number(manifest.architecture?.numLayers ?? 0),
|
|
@@ -204,12 +304,49 @@ export class InferencePipeline extends PipelineState {
|
|
|
204
304
|
this.runtimeConfig = applyModelBatchingRuntimeDefaults(
|
|
205
305
|
this.runtimeConfig,
|
|
206
306
|
manifest,
|
|
207
|
-
this.modelConfig
|
|
307
|
+
this.modelConfig,
|
|
308
|
+
this.runtimeOverrides
|
|
208
309
|
);
|
|
209
310
|
this.useTiedEmbeddings = this.modelConfig.useTiedEmbeddings;
|
|
210
311
|
this.embeddingVocabSize = this.modelConfig.embeddingVocabSize;
|
|
211
312
|
this.embeddingTranspose = this.modelConfig.embeddingTranspose;
|
|
212
313
|
|
|
314
|
+
// Vision capability detection — gated by manifest fields
|
|
315
|
+
const imageTokenId = manifest.image_token_id;
|
|
316
|
+
const hasVisionQuant = manifest.quantizationInfo?.vision != null;
|
|
317
|
+
if (Number.isInteger(imageTokenId) && imageTokenId > 0 && hasVisionQuant) {
|
|
318
|
+
this.visionCapable = true;
|
|
319
|
+
this.imageTokenId = imageTokenId;
|
|
320
|
+
this.visionConfig = this.modelConfig.visionConfig;
|
|
321
|
+
if (!this.visionConfig) {
|
|
322
|
+
throw new Error(
|
|
323
|
+
`Manifest declares image_token_id=${imageTokenId} and quantizationInfo.vision ` +
|
|
324
|
+
'but no vision_config was resolved. Check conversion config.'
|
|
325
|
+
);
|
|
326
|
+
}
|
|
327
|
+
log.info('Pipeline', `Vision capable: imageTokenId=${imageTokenId}`);
|
|
328
|
+
} else {
|
|
329
|
+
this.visionCapable = false;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
// Audio capability detection — gated by manifest fields
|
|
333
|
+
const audioTokenId = manifest.audio_token_id;
|
|
334
|
+
const hasAudioQuant = manifest.quantizationInfo?.audio != null;
|
|
335
|
+
if (Number.isInteger(audioTokenId) && audioTokenId > 0 && hasAudioQuant) {
|
|
336
|
+
this.audioCapable = true;
|
|
337
|
+
this.audioTokenId = audioTokenId;
|
|
338
|
+
this.audioConfig = this.modelConfig.audioConfig;
|
|
339
|
+
if (!this.audioConfig) {
|
|
340
|
+
throw new Error(
|
|
341
|
+
`Manifest declares audio_token_id=${audioTokenId} and quantizationInfo.audio ` +
|
|
342
|
+
'but no audio_config was resolved. Check conversion config.'
|
|
343
|
+
);
|
|
344
|
+
}
|
|
345
|
+
log.info('Pipeline', `Audio capable: audioTokenId=${audioTokenId}`);
|
|
346
|
+
} else {
|
|
347
|
+
this.audioCapable = false;
|
|
348
|
+
}
|
|
349
|
+
|
|
213
350
|
await runKernelWarmup({
|
|
214
351
|
useGPU: this.useGPU,
|
|
215
352
|
kernelWarmup: this.runtimeConfig.shared?.kernelWarmup,
|
|
@@ -222,6 +359,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
222
359
|
const kernelPathState = resolveKernelPathState({
|
|
223
360
|
manifest,
|
|
224
361
|
runtimeConfig: this.runtimeConfig,
|
|
362
|
+
runtimeOverrides: this.runtimeOverrides,
|
|
225
363
|
modelConfig: this.modelConfig,
|
|
226
364
|
});
|
|
227
365
|
this.resolvedKernelPath = kernelPathState.resolvedKernelPath;
|
|
@@ -254,7 +392,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
254
392
|
// Check for execution-v1 kvDtype conflict with manifest quantization info
|
|
255
393
|
if (this.executionV1State && this.resolvedKernelPath) {
|
|
256
394
|
const manifestComputeHint = manifest?.quantizationInfo?.compute;
|
|
257
|
-
const resolvedKvDtype = this.runtimeConfig.inference.kvcache?.kvDtype;
|
|
395
|
+
const resolvedKvDtype = this.runtimeConfig.inference.session?.kvcache?.kvDtype;
|
|
258
396
|
if (
|
|
259
397
|
manifestComputeHint
|
|
260
398
|
&& resolvedKvDtype
|
|
@@ -270,7 +408,16 @@ export class InferencePipeline extends PipelineState {
|
|
|
270
408
|
}
|
|
271
409
|
|
|
272
410
|
// Initialize KV cache
|
|
273
|
-
|
|
411
|
+
if (this.modelConfig.decodeStrategy === 'replay_prefill') {
|
|
412
|
+
this.kvCache = null;
|
|
413
|
+
log.warn(
|
|
414
|
+
'Pipeline',
|
|
415
|
+
'Replay-prefill decode enabled for this model. Incremental KV-cache decode is disabled ' +
|
|
416
|
+
'because the model config did not resolve explicit layerTypes for mixed-geometry/shared-KV decode.'
|
|
417
|
+
);
|
|
418
|
+
} else {
|
|
419
|
+
this.kvCache = createKVCache(this.modelConfig, this.useGPU, this.debug, this.runtimeConfig.inference);
|
|
420
|
+
}
|
|
274
421
|
this.executionPlanState = compileExecutionPlanState({
|
|
275
422
|
runtimeConfig: this.runtimeConfig,
|
|
276
423
|
resolvedKernelPath: this.resolvedKernelPath,
|
|
@@ -344,6 +491,8 @@ export class InferencePipeline extends PipelineState {
|
|
|
344
491
|
kernelPathSource: this.kernelPathSource,
|
|
345
492
|
keepF32Weights: this.runtimeConfig.inference.compute.keepF32Weights === true,
|
|
346
493
|
loaderDebug: this.runtimeConfig?.shared?.debug?.loader ?? null,
|
|
494
|
+
perLayerInputSession:
|
|
495
|
+
this.runtimeConfig.inference.session?.perLayerInputs ?? this.modelConfig.perLayerInputsSession,
|
|
347
496
|
onProgress: (info) => {
|
|
348
497
|
if (info.stage !== 'layers' && info.stage !== 'shards') {
|
|
349
498
|
log.verbose('Loader', `${info.stage}: ${Math.round(info.progress * 100)}%${info.message ? ` - ${info.message}` : ''}`);
|
|
@@ -365,6 +514,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
365
514
|
this.weights.set('embed', result.embeddings);
|
|
366
515
|
this.weights.set('lm_head', result.lmHead);
|
|
367
516
|
this.weights.set('final_norm', result.finalNorm);
|
|
517
|
+
this.weights.set('per_layer_inputs', result.perLayerInputWeights);
|
|
368
518
|
this.embeddingPostprocessor = result.embeddingPostprocessor;
|
|
369
519
|
|
|
370
520
|
this.layerRouterWeights = result.layerRouterWeights;
|
|
@@ -380,7 +530,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
380
530
|
}
|
|
381
531
|
|
|
382
532
|
if (this.useGPU && this.modelConfig) {
|
|
383
|
-
fuseQKVWeights(result.layerWeights, this.modelConfig);
|
|
533
|
+
fuseQKVWeights(result.layerWeights, this.modelConfig, this.resolvedKernelPath);
|
|
384
534
|
}
|
|
385
535
|
|
|
386
536
|
if (this.useGPU && this.modelConfig) {
|
|
@@ -388,7 +538,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
388
538
|
try {
|
|
389
539
|
this.decodeBuffers?.ensureBuffers({
|
|
390
540
|
hiddenSize: this.modelConfig.hiddenSize,
|
|
391
|
-
intermediateSize: this.modelConfig.
|
|
541
|
+
intermediateSize: this.modelConfig.maxIntermediateSize,
|
|
392
542
|
activationDtype: activeExecutionPlan.activationDtype,
|
|
393
543
|
enablePingPong: true,
|
|
394
544
|
});
|
|
@@ -422,14 +572,19 @@ export class InferencePipeline extends PipelineState {
|
|
|
422
572
|
const config = (this.modelConfig);
|
|
423
573
|
const maxSeqLen = config.maxSeqLen;
|
|
424
574
|
const ropeBuffers = await initRoPEFrequencies({
|
|
425
|
-
headDim: config.headDim,
|
|
575
|
+
headDim: config.globalHeadDim ?? config.headDim,
|
|
576
|
+
localHeadDim: config.headDim,
|
|
426
577
|
rotaryDim: config.ropeRotaryDim,
|
|
578
|
+
ropeLocalRotaryDim: config.ropeLocalRotaryDim,
|
|
579
|
+
ropeFrequencyBaseDim: config.ropeFrequencyBaseDim,
|
|
580
|
+
ropeLocalFrequencyBaseDim: config.ropeLocalFrequencyBaseDim,
|
|
427
581
|
maxSeqLen,
|
|
428
582
|
ropeTheta: config.ropeTheta,
|
|
429
583
|
ropeLocalTheta: config.ropeLocalTheta,
|
|
430
584
|
mropeInterleaved: config.mropeInterleaved,
|
|
431
585
|
mropeSection: config.mropeSection,
|
|
432
586
|
partialRotaryFactor: config.partialRotaryFactor,
|
|
587
|
+
ropeLocalPartialRotaryFactor: config.ropeLocalPartialRotaryFactor,
|
|
433
588
|
ropeScale: config.ropeScale,
|
|
434
589
|
ropeLocalScale: config.ropeLocalScale,
|
|
435
590
|
ropeScalingType: config.ropeScalingType,
|
|
@@ -489,6 +644,667 @@ export class InferencePipeline extends PipelineState {
|
|
|
489
644
|
}
|
|
490
645
|
|
|
491
646
|
|
|
647
|
+
async _loadVisionWeights() {
|
|
648
|
+
const loader = this.dopplerLoader ?? getDopplerLoader(this.runtimeConfig.loading);
|
|
649
|
+
const vc = this.visionConfig;
|
|
650
|
+
const depth = vc.depth;
|
|
651
|
+
|
|
652
|
+
const loadRequiredTensor = async (name, toGPU = true) => {
|
|
653
|
+
const tensor = await loader.loadTensor(name, toGPU, true);
|
|
654
|
+
if (!tensor) {
|
|
655
|
+
throw new Error(`Vision tensor "${name}" is missing from the converted artifact.`);
|
|
656
|
+
}
|
|
657
|
+
return tensor;
|
|
658
|
+
};
|
|
659
|
+
const loadScalar = async (name) => {
|
|
660
|
+
const tensor = await loadRequiredTensor(name, false);
|
|
661
|
+
if (tensor instanceof Float32Array) {
|
|
662
|
+
if (tensor.length !== 1) {
|
|
663
|
+
throw new Error(`Vision scalar "${name}" must be a single-element tensor, got length=${tensor.length}.`);
|
|
664
|
+
}
|
|
665
|
+
return tensor[0];
|
|
666
|
+
}
|
|
667
|
+
if (ArrayBuffer.isView(tensor) && tensor.length === 1) {
|
|
668
|
+
return Number(tensor[0]);
|
|
669
|
+
}
|
|
670
|
+
if (typeof tensor === 'number') {
|
|
671
|
+
return tensor;
|
|
672
|
+
}
|
|
673
|
+
throw new Error(
|
|
674
|
+
`Vision scalar "${name}" must decode to a single numeric value, ` +
|
|
675
|
+
`got ${tensor?.constructor?.name ?? typeof tensor} length=${tensor?.length ?? 'N/A'}.`
|
|
676
|
+
);
|
|
677
|
+
};
|
|
678
|
+
const loadClipRange = async (prefix) => ({
|
|
679
|
+
inputMin: await loadScalar(`${prefix}.input_min`),
|
|
680
|
+
inputMax: await loadScalar(`${prefix}.input_max`),
|
|
681
|
+
outputMin: await loadScalar(`${prefix}.output_min`),
|
|
682
|
+
outputMax: await loadScalar(`${prefix}.output_max`),
|
|
683
|
+
});
|
|
684
|
+
|
|
685
|
+
if (vc.visionArchitecture === 'gemma4') {
|
|
686
|
+
const visionWeights = {
|
|
687
|
+
textHiddenSize: this.modelConfig.hiddenSize,
|
|
688
|
+
patchInputProj: await loadRequiredTensor('model.vision_tower.patch_embedder.input_proj.weight'),
|
|
689
|
+
patchPositionEmbeddingTable: await loadRequiredTensor('model.vision_tower.patch_embedder.position_embedding_table', false),
|
|
690
|
+
projector: await loadRequiredTensor('model.embed_vision.embedding_projection.weight'),
|
|
691
|
+
layers: [],
|
|
692
|
+
};
|
|
693
|
+
|
|
694
|
+
if (!(visionWeights.patchPositionEmbeddingTable instanceof Float32Array)) {
|
|
695
|
+
throw new Error(
|
|
696
|
+
'Gemma 4 vision position_embedding_table must decode to Float32Array on CPU. ' +
|
|
697
|
+
'Re-convert the artifact if this tensor was quantized incorrectly.'
|
|
698
|
+
);
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
for (let i = 0; i < depth; i++) {
|
|
702
|
+
const prefix = `model.vision_tower.encoder.layers.${i}`;
|
|
703
|
+
const attnPrefix = `${prefix}.self_attn`;
|
|
704
|
+
const mlpPrefix = `${prefix}.mlp`;
|
|
705
|
+
visionWeights.layers.push({
|
|
706
|
+
inputLayerNorm: await loadRequiredTensor(`${prefix}.input_layernorm.weight`),
|
|
707
|
+
postAttentionLayerNorm: await loadRequiredTensor(`${prefix}.post_attention_layernorm.weight`),
|
|
708
|
+
preFeedforwardLayerNorm: await loadRequiredTensor(`${prefix}.pre_feedforward_layernorm.weight`),
|
|
709
|
+
postFeedforwardLayerNorm: await loadRequiredTensor(`${prefix}.post_feedforward_layernorm.weight`),
|
|
710
|
+
qNorm: await loadRequiredTensor(`${attnPrefix}.q_norm.weight`),
|
|
711
|
+
kNorm: await loadRequiredTensor(`${attnPrefix}.k_norm.weight`),
|
|
712
|
+
qProj: await loadRequiredTensor(`${attnPrefix}.q_proj.linear.weight`),
|
|
713
|
+
kProj: await loadRequiredTensor(`${attnPrefix}.k_proj.linear.weight`),
|
|
714
|
+
vProj: await loadRequiredTensor(`${attnPrefix}.v_proj.linear.weight`),
|
|
715
|
+
oProj: await loadRequiredTensor(`${attnPrefix}.o_proj.linear.weight`),
|
|
716
|
+
qProjClip: await loadClipRange(`${attnPrefix}.q_proj`),
|
|
717
|
+
kProjClip: await loadClipRange(`${attnPrefix}.k_proj`),
|
|
718
|
+
vProjClip: await loadClipRange(`${attnPrefix}.v_proj`),
|
|
719
|
+
oProjClip: await loadClipRange(`${attnPrefix}.o_proj`),
|
|
720
|
+
gateProj: await loadRequiredTensor(`${mlpPrefix}.gate_proj.linear.weight`),
|
|
721
|
+
upProj: await loadRequiredTensor(`${mlpPrefix}.up_proj.linear.weight`),
|
|
722
|
+
downProj: await loadRequiredTensor(`${mlpPrefix}.down_proj.linear.weight`),
|
|
723
|
+
gateProjClip: await loadClipRange(`${mlpPrefix}.gate_proj`),
|
|
724
|
+
upProjClip: await loadClipRange(`${mlpPrefix}.up_proj`),
|
|
725
|
+
downProjClip: await loadClipRange(`${mlpPrefix}.down_proj`),
|
|
726
|
+
});
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
this.visionWeights = visionWeights;
|
|
730
|
+
log.info('Pipeline', `Vision weights loaded (${depth} Gemma 4 encoder layers)`);
|
|
731
|
+
return;
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
const visionWeights = {};
|
|
735
|
+
|
|
736
|
+
// Patch embedding weights
|
|
737
|
+
const patchProjName = 'visual.patch_embed.proj.weight';
|
|
738
|
+
const patchProjBiasName = 'visual.patch_embed.proj.bias';
|
|
739
|
+
visionWeights.patchProjWeight = await loader.loadTensor(patchProjName, true, true);
|
|
740
|
+
visionWeights.patchProjBias = await loader.loadTensor(patchProjBiasName, true, true);
|
|
741
|
+
|
|
742
|
+
// Vision encoder layer weights
|
|
743
|
+
visionWeights.layers = [];
|
|
744
|
+
for (let i = 0; i < depth; i++) {
|
|
745
|
+
const prefix = `visual.blocks.${i}`;
|
|
746
|
+
const layerW = {
|
|
747
|
+
norm1Weight: await loader.loadTensor(`${prefix}.norm1.weight`, true, true),
|
|
748
|
+
norm2Weight: await loader.loadTensor(`${prefix}.norm2.weight`, true, true),
|
|
749
|
+
qkvWeight: await loader.loadTensor(`${prefix}.attn.qkv.weight`, true, true),
|
|
750
|
+
qkvBias: await loader.loadTensor(`${prefix}.attn.qkv.bias`, true, true),
|
|
751
|
+
projWeight: await loader.loadTensor(`${prefix}.attn.proj.weight`, true, true),
|
|
752
|
+
projBias: await loader.loadTensor(`${prefix}.attn.proj.bias`, true, true),
|
|
753
|
+
fc1Weight: await loader.loadTensor(`${prefix}.mlp.fc1.weight`, true, true),
|
|
754
|
+
fc1Bias: await loader.loadTensor(`${prefix}.mlp.fc1.bias`, true, true),
|
|
755
|
+
fc2Weight: await loader.loadTensor(`${prefix}.mlp.fc2.weight`, true, true),
|
|
756
|
+
fc2Bias: await loader.loadTensor(`${prefix}.mlp.fc2.bias`, true, true),
|
|
757
|
+
};
|
|
758
|
+
visionWeights.layers.push(layerW);
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
// Spatial merge projection
|
|
762
|
+
visionWeights.mergerLnWeight = await loader.loadTensor('visual.merger.ln_q.weight', true, true);
|
|
763
|
+
visionWeights.mergerMlp0Weight = await loader.loadTensor('visual.merger.mlp.0.weight', true, true);
|
|
764
|
+
visionWeights.mergerMlp0Bias = await loader.loadTensor('visual.merger.mlp.0.bias', true, true);
|
|
765
|
+
visionWeights.mergerMlp2Weight = await loader.loadTensor('visual.merger.mlp.2.weight', true, true);
|
|
766
|
+
visionWeights.mergerMlp2Bias = await loader.loadTensor('visual.merger.mlp.2.bias', true, true);
|
|
767
|
+
|
|
768
|
+
this.visionWeights = visionWeights;
|
|
769
|
+
log.info('Pipeline', `Vision weights loaded (${depth} encoder layers)`);
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
async _ensureVisionWeightsLoaded() {
|
|
773
|
+
if (!this.visionCapable) {
|
|
774
|
+
throw new Error(
|
|
775
|
+
'Pipeline does not support vision weights (no image_token_id in manifest).'
|
|
776
|
+
);
|
|
777
|
+
}
|
|
778
|
+
if (this.visionWeights) {
|
|
779
|
+
return;
|
|
780
|
+
}
|
|
781
|
+
log.info('Pipeline', 'Loading vision weights on demand');
|
|
782
|
+
await this._loadVisionWeights();
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
async _loadAudioWeights() {
|
|
787
|
+
const loader = this.dopplerLoader ?? getDopplerLoader(this.runtimeConfig.loading);
|
|
788
|
+
const ac = this.audioConfig;
|
|
789
|
+
const depth = ac.depth;
|
|
790
|
+
|
|
791
|
+
const loadRequiredTensor = async (name, toGPU = true) => {
|
|
792
|
+
const tensor = await loader.loadTensor(name, toGPU, true);
|
|
793
|
+
if (!tensor) {
|
|
794
|
+
throw new Error(`Audio tensor "${name}" is missing from the converted artifact.`);
|
|
795
|
+
}
|
|
796
|
+
return tensor;
|
|
797
|
+
};
|
|
798
|
+
const loadScalar = async (name) => {
|
|
799
|
+
const tensor = await loadRequiredTensor(name, false);
|
|
800
|
+
if (tensor instanceof Float32Array) {
|
|
801
|
+
if (tensor.length !== 1) {
|
|
802
|
+
throw new Error(`Audio scalar "${name}" must be a single-element tensor, got length=${tensor.length}.`);
|
|
803
|
+
}
|
|
804
|
+
return tensor[0];
|
|
805
|
+
}
|
|
806
|
+
if (ArrayBuffer.isView(tensor) && tensor.length === 1) {
|
|
807
|
+
return Number(tensor[0]);
|
|
808
|
+
}
|
|
809
|
+
if (typeof tensor === 'number') {
|
|
810
|
+
return tensor;
|
|
811
|
+
}
|
|
812
|
+
throw new Error(
|
|
813
|
+
`Audio scalar "${name}" must decode to a single numeric value, ` +
|
|
814
|
+
`got ${tensor?.constructor?.name ?? typeof tensor} length=${tensor?.length ?? 'N/A'}.`
|
|
815
|
+
);
|
|
816
|
+
};
|
|
817
|
+
const loadClipRange = async (prefix) => ({
|
|
818
|
+
inputMin: await loadScalar(`${prefix}.input_min`),
|
|
819
|
+
inputMax: await loadScalar(`${prefix}.input_max`),
|
|
820
|
+
outputMin: await loadScalar(`${prefix}.output_min`),
|
|
821
|
+
outputMax: await loadScalar(`${prefix}.output_max`),
|
|
822
|
+
});
|
|
823
|
+
|
|
824
|
+
const audioWeights = {
|
|
825
|
+
// Subsampling
|
|
826
|
+
subsampleConv0Weight: await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer0.conv.weight'),
|
|
827
|
+
subsampleNorm0Weight: await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer0.norm.weight'),
|
|
828
|
+
subsampleConv1Weight: await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer1.conv.weight'),
|
|
829
|
+
subsampleNorm1Weight: await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer1.norm.weight'),
|
|
830
|
+
subsampleInputProjWeight: await loadRequiredTensor('model.audio_tower.subsample_conv_projection.input_proj_linear.weight'),
|
|
831
|
+
// Output
|
|
832
|
+
outputProjWeight: await loadRequiredTensor('model.audio_tower.output_proj.weight'),
|
|
833
|
+
outputProjBias: await loadRequiredTensor('model.audio_tower.output_proj.bias'),
|
|
834
|
+
audioEmbeddingProjWeight: await loadRequiredTensor('model.embed_audio.embedding_projection.weight'),
|
|
835
|
+
layers: [],
|
|
836
|
+
};
|
|
837
|
+
|
|
838
|
+
for (let i = 0; i < depth; i++) {
|
|
839
|
+
const prefix = `model.audio_tower.layers.${i}`;
|
|
840
|
+
const layer = {
|
|
841
|
+
// Feed-forward 1 (Macaron half-step)
|
|
842
|
+
feedForward1: {
|
|
843
|
+
preLayerNorm: await loadRequiredTensor(`${prefix}.feed_forward1.pre_layer_norm.weight`),
|
|
844
|
+
ffwLayer1Weight: await loadRequiredTensor(`${prefix}.feed_forward1.ffw_layer_1.linear.weight`),
|
|
845
|
+
ffwLayer1Clip: await loadClipRange(`${prefix}.feed_forward1.ffw_layer_1`),
|
|
846
|
+
ffwLayer2Weight: await loadRequiredTensor(`${prefix}.feed_forward1.ffw_layer_2.linear.weight`),
|
|
847
|
+
ffwLayer2Clip: await loadClipRange(`${prefix}.feed_forward1.ffw_layer_2`),
|
|
848
|
+
postLayerNorm: await loadRequiredTensor(`${prefix}.feed_forward1.post_layer_norm.weight`),
|
|
849
|
+
},
|
|
850
|
+
// Self-attention
|
|
851
|
+
normPreAttn: await loadRequiredTensor(`${prefix}.norm_pre_attn.weight`),
|
|
852
|
+
qProj: await loadRequiredTensor(`${prefix}.self_attn.q_proj.linear.weight`),
|
|
853
|
+
qProjClip: await loadClipRange(`${prefix}.self_attn.q_proj`),
|
|
854
|
+
kProj: await loadRequiredTensor(`${prefix}.self_attn.k_proj.linear.weight`),
|
|
855
|
+
kProjClip: await loadClipRange(`${prefix}.self_attn.k_proj`),
|
|
856
|
+
vProj: await loadRequiredTensor(`${prefix}.self_attn.v_proj.linear.weight`),
|
|
857
|
+
vProjClip: await loadClipRange(`${prefix}.self_attn.v_proj`),
|
|
858
|
+
perDimScale: await loadRequiredTensor(`${prefix}.self_attn.per_dim_scale`),
|
|
859
|
+
relativeKProj: await loadRequiredTensor(`${prefix}.self_attn.relative_k_proj.weight`),
|
|
860
|
+
postProj: await loadRequiredTensor(`${prefix}.self_attn.post.linear.weight`),
|
|
861
|
+
postProjClip: await loadClipRange(`${prefix}.self_attn.post`),
|
|
862
|
+
normPostAttn: await loadRequiredTensor(`${prefix}.norm_post_attn.weight`),
|
|
863
|
+
// Convolution module (LConv1D)
|
|
864
|
+
lconvPreLayerNorm: await loadRequiredTensor(`${prefix}.lconv1d.pre_layer_norm.weight`),
|
|
865
|
+
lconvLinearStartWeight: await loadRequiredTensor(`${prefix}.lconv1d.linear_start.linear.weight`),
|
|
866
|
+
lconvLinearStartClip: await loadClipRange(`${prefix}.lconv1d.linear_start`),
|
|
867
|
+
lconvDepthwiseWeight: await loadRequiredTensor(`${prefix}.lconv1d.depthwise_conv1d.weight`),
|
|
868
|
+
lconvConvNorm: await loadRequiredTensor(`${prefix}.lconv1d.conv_norm.weight`),
|
|
869
|
+
lconvLinearEndWeight: await loadRequiredTensor(`${prefix}.lconv1d.linear_end.linear.weight`),
|
|
870
|
+
lconvLinearEndClip: await loadClipRange(`${prefix}.lconv1d.linear_end`),
|
|
871
|
+
// Feed-forward 2 (Macaron half-step)
|
|
872
|
+
feedForward2: {
|
|
873
|
+
preLayerNorm: await loadRequiredTensor(`${prefix}.feed_forward2.pre_layer_norm.weight`),
|
|
874
|
+
ffwLayer1Weight: await loadRequiredTensor(`${prefix}.feed_forward2.ffw_layer_1.linear.weight`),
|
|
875
|
+
ffwLayer1Clip: await loadClipRange(`${prefix}.feed_forward2.ffw_layer_1`),
|
|
876
|
+
ffwLayer2Weight: await loadRequiredTensor(`${prefix}.feed_forward2.ffw_layer_2.linear.weight`),
|
|
877
|
+
ffwLayer2Clip: await loadClipRange(`${prefix}.feed_forward2.ffw_layer_2`),
|
|
878
|
+
postLayerNorm: await loadRequiredTensor(`${prefix}.feed_forward2.post_layer_norm.weight`),
|
|
879
|
+
},
|
|
880
|
+
// Final layer norm
|
|
881
|
+
normOut: await loadRequiredTensor(`${prefix}.norm_out.weight`),
|
|
882
|
+
};
|
|
883
|
+
audioWeights.layers.push(layer);
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
this.audioWeights = audioWeights;
|
|
887
|
+
log.info('Pipeline', `Audio weights loaded (${depth} conformer layers)`);
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
async _ensureAudioWeightsLoaded() {
|
|
891
|
+
if (!this.audioCapable) {
|
|
892
|
+
throw new Error(
|
|
893
|
+
'Pipeline does not support audio weights (no audio_token_id in manifest).'
|
|
894
|
+
);
|
|
895
|
+
}
|
|
896
|
+
if (this.audioWeights) {
|
|
897
|
+
return;
|
|
898
|
+
}
|
|
899
|
+
log.info('Pipeline', 'Loading audio weights on demand');
|
|
900
|
+
await this._loadAudioWeights();
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
|
|
904
|
+
// ==========================================================================
|
|
905
|
+
// Vision: transcribeImage
|
|
906
|
+
// ==========================================================================
|
|
907
|
+
|
|
908
|
+
/**
|
|
909
|
+
* Transcribe text from an image using the vision encoder and text decoder.
|
|
910
|
+
*
|
|
911
|
+
* @param {object} params
|
|
912
|
+
* @param {Uint8Array|Float32Array} params.imageBytes Raw image pixel data
|
|
913
|
+
* @param {number} params.width Image width
|
|
914
|
+
* @param {number} params.height Image height
|
|
915
|
+
* @param {string} [params.prompt] Custom transcription prompt
|
|
916
|
+
* @param {number} [params.maxTokens] Max tokens to generate
|
|
917
|
+
* @param {number} [params.softTokenBudget] Per-request soft token budget (Gemma 4 tiers: 70/140/280/560/1120)
|
|
918
|
+
* @returns {Promise<{ text: string, tokens: number[] }>}
|
|
919
|
+
*/
|
|
920
|
+
async transcribeImage({ imageBytes, width, height, prompt, maxTokens, softTokenBudget }) {
|
|
921
|
+
if (!this.visionCapable) {
|
|
922
|
+
throw new Error(
|
|
923
|
+
'Pipeline does not support image transcription (no image_token_id in manifest).'
|
|
924
|
+
);
|
|
925
|
+
}
|
|
926
|
+
await this._ensureVisionWeightsLoaded();
|
|
927
|
+
|
|
928
|
+
this.reset();
|
|
929
|
+
|
|
930
|
+
// Lazy-load vision module (avoids GPU kernel dependency for text-only pipelines)
|
|
931
|
+
const { encodeImage } = await import('./vision/index.js');
|
|
932
|
+
|
|
933
|
+
// Step 1: Encode image through vision pipeline
|
|
934
|
+
const encodeResult = await encodeImage({
|
|
935
|
+
pixels: imageBytes,
|
|
936
|
+
width,
|
|
937
|
+
height,
|
|
938
|
+
visionConfig: this.visionConfig,
|
|
939
|
+
weights: this.visionWeights,
|
|
940
|
+
softTokenBudget,
|
|
941
|
+
});
|
|
942
|
+
|
|
943
|
+
// Step 2: Build the multimodal prompt from the model's chat template and
|
|
944
|
+
// expand the single <|image|> placeholder into the exact visual-token span.
|
|
945
|
+
const requestedPrompt = prompt ?? 'Describe the image in one short sentence.';
|
|
946
|
+
const imageTokenId = this.visionConfig?.imageTokenId ?? this.modelConfig?.imageTokenId;
|
|
947
|
+
if (imageTokenId == null) {
|
|
948
|
+
throw new Error(
|
|
949
|
+
'Pipeline missing image_token_id. Re-convert the model with image token metadata.'
|
|
950
|
+
);
|
|
951
|
+
}
|
|
952
|
+
if (this.visionConfig?.visionArchitecture !== 'gemma4') {
|
|
953
|
+
throw new Error(
|
|
954
|
+
`[Pipeline] transcribeImage: unsupported vision architecture "${this.visionConfig?.visionArchitecture ?? 'unknown'}". ` +
|
|
955
|
+
'This runtime path currently requires Gemma 4 multimodal prompt expansion.'
|
|
956
|
+
);
|
|
957
|
+
}
|
|
958
|
+
const templateType = this.modelConfig?.chatTemplateType ?? 'gemma4';
|
|
959
|
+
const chatOptions = this.modelConfig?.chatTemplateThinking === true ? { thinking: true } : undefined;
|
|
960
|
+
const multimodalPrompt = formatChatMessages([
|
|
961
|
+
{
|
|
962
|
+
role: 'user',
|
|
963
|
+
content: [
|
|
964
|
+
{ type: 'image' },
|
|
965
|
+
{ type: 'text', text: requestedPrompt },
|
|
966
|
+
],
|
|
967
|
+
},
|
|
968
|
+
], templateType, chatOptions);
|
|
969
|
+
const promptTokenIds = this.tokenizer.encode(multimodalPrompt);
|
|
970
|
+
const imageTokenSpanLength = encodeResult.numTokens;
|
|
971
|
+
const effectiveBudget = softTokenBudget ?? this.visionConfig?.defaultOutputLength;
|
|
972
|
+
const maxImageTokenSpanLength = Number(effectiveBudget);
|
|
973
|
+
if (!Number.isFinite(maxImageTokenSpanLength) || maxImageTokenSpanLength < 1 || Math.floor(maxImageTokenSpanLength) !== maxImageTokenSpanLength) {
|
|
974
|
+
throw new Error(
|
|
975
|
+
`[Pipeline] transcribeImage: invalid soft token budget ${effectiveBudget}. ` +
|
|
976
|
+
'Expected a positive integer from the resolved vision config or softTokenBudget parameter.'
|
|
977
|
+
);
|
|
978
|
+
}
|
|
979
|
+
if (imageTokenSpanLength > maxImageTokenSpanLength) {
|
|
980
|
+
throw new Error(
|
|
981
|
+
`[Pipeline] transcribeImage: encoded Gemma 4 image produced ${imageTokenSpanLength} soft tokens, ` +
|
|
982
|
+
`which exceeds the effective soft token budget=${maxImageTokenSpanLength}.`
|
|
983
|
+
);
|
|
984
|
+
}
|
|
985
|
+
const boiTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<|image>', 'Gemma 4 BOI token');
|
|
986
|
+
const eoiTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<image|>', 'Gemma 4 EOI token');
|
|
987
|
+
const { inputIds: fullTokenIds, imageStartOffset } = expandImagePlaceholderTokenIds(
|
|
988
|
+
promptTokenIds,
|
|
989
|
+
imageTokenId,
|
|
990
|
+
imageTokenSpanLength,
|
|
991
|
+
{ boiTokenId, eoiTokenId }
|
|
992
|
+
);
|
|
993
|
+
const padTokenId = this.tokenizer?.getSpecialTokens?.()?.pad;
|
|
994
|
+
if (!Number.isFinite(padTokenId) || Math.floor(padTokenId) !== padTokenId || padTokenId < 0) {
|
|
995
|
+
throw new Error(
|
|
996
|
+
`[Pipeline] transcribeImage: Gemma 4 multimodal prefill requires a tokenizer pad token ID, got ${padTokenId}.`
|
|
997
|
+
);
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
// Step 3: Generate with embedding override at the image token offset.
|
|
1001
|
+
const tokens = [];
|
|
1002
|
+
const maxGen = maxTokens ?? 512;
|
|
1003
|
+
const stopTokenIds = this.modelConfig.stopTokenIds;
|
|
1004
|
+
|
|
1005
|
+
try {
|
|
1006
|
+
const generation = await this.generator.generateTokenIds('', buildConservativeMultimodalGenerationOptions({
|
|
1007
|
+
inputIds: fullTokenIds,
|
|
1008
|
+
embeddingOverrides: {
|
|
1009
|
+
prefixLength: encodeResult.numTokens,
|
|
1010
|
+
offset: imageStartOffset,
|
|
1011
|
+
embeddings: encodeResult.features,
|
|
1012
|
+
},
|
|
1013
|
+
__internalEmbeddingInputSpan: {
|
|
1014
|
+
offset: imageStartOffset,
|
|
1015
|
+
length: encodeResult.numTokens,
|
|
1016
|
+
tokenId: padTokenId,
|
|
1017
|
+
},
|
|
1018
|
+
__internalMultimodalBidirectionalSpan: {
|
|
1019
|
+
offset: imageStartOffset,
|
|
1020
|
+
length: encodeResult.numTokens,
|
|
1021
|
+
},
|
|
1022
|
+
maxTokens: maxGen,
|
|
1023
|
+
temperature: 0,
|
|
1024
|
+
topK: 1,
|
|
1025
|
+
topP: 1,
|
|
1026
|
+
repetitionPenalty: 1,
|
|
1027
|
+
}));
|
|
1028
|
+
for (const token of generation.tokenIds ?? []) {
|
|
1029
|
+
if (Array.isArray(stopTokenIds) && stopTokenIds.includes(token)) break;
|
|
1030
|
+
tokens.push(token);
|
|
1031
|
+
}
|
|
1032
|
+
} finally {
|
|
1033
|
+
if (encodeResult.features) {
|
|
1034
|
+
releaseBuffer(encodeResult.features);
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
const text = this.tokenizer.decode(tokens);
|
|
1039
|
+
return { text, tokens };
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
|
|
1043
|
+
// ==========================================================================
|
|
1044
|
+
// Video: transcribeVideo
|
|
1045
|
+
// ==========================================================================
|
|
1046
|
+
|
|
1047
|
+
/**
|
|
1048
|
+
* Transcribe text from video using the vision encoder (per-frame) and text decoder.
|
|
1049
|
+
*
|
|
1050
|
+
* Video is processed as sampled frames through the existing vision encoder.
|
|
1051
|
+
* Each frame is encoded independently and the visual tokens are concatenated.
|
|
1052
|
+
*
|
|
1053
|
+
* @param {object} params
|
|
1054
|
+
* @param {Array<{ pixels: Uint8Array|Float32Array, width: number, height: number }>} params.frames Decoded video frames
|
|
1055
|
+
* @param {string} [params.prompt] Custom transcription prompt
|
|
1056
|
+
* @param {number} [params.maxTokens] Max tokens to generate
|
|
1057
|
+
* @param {number} [params.maxFrames=8] Maximum frames to sample
|
|
1058
|
+
* @param {number} [params.perFrameSoftTokenBudget] Soft token budget per frame
|
|
1059
|
+
* @returns {Promise<{ text: string, tokens: number[] }>}
|
|
1060
|
+
*/
|
|
1061
|
+
async transcribeVideo({ frames, prompt, maxTokens, maxFrames, perFrameSoftTokenBudget }) {
|
|
1062
|
+
if (!this.visionCapable) {
|
|
1063
|
+
throw new Error(
|
|
1064
|
+
'Pipeline does not support video transcription (no image_token_id in manifest for vision encoder).'
|
|
1065
|
+
);
|
|
1066
|
+
}
|
|
1067
|
+
await this._ensureVisionWeightsLoaded();
|
|
1068
|
+
|
|
1069
|
+
this.reset();
|
|
1070
|
+
|
|
1071
|
+
// Lazy-load video module
|
|
1072
|
+
const { encodeVideo } = await import('./video/index.js');
|
|
1073
|
+
|
|
1074
|
+
// Step 1: Encode video frames through vision pipeline
|
|
1075
|
+
const encodeResult = await encodeVideo({
|
|
1076
|
+
frames,
|
|
1077
|
+
visionConfig: this.visionConfig,
|
|
1078
|
+
weights: this.visionWeights,
|
|
1079
|
+
maxFrames: maxFrames ?? 8,
|
|
1080
|
+
perFrameSoftTokenBudget,
|
|
1081
|
+
});
|
|
1082
|
+
|
|
1083
|
+
// Step 2: Build the multimodal prompt with <|video|> placeholder
|
|
1084
|
+
const requestedPrompt = prompt ?? 'Describe the video in one short sentence.';
|
|
1085
|
+
const videoTokenId = this.tokenizer?.model?.tokenToId?.('<|video_token|>')
|
|
1086
|
+
?? this.tokenizer?.encode?.('<|video|>')?.find?.((id) => id !== undefined)
|
|
1087
|
+
?? null;
|
|
1088
|
+
// Fall back to image token ID for video placeholder expansion
|
|
1089
|
+
const placeholderTokenId = videoTokenId ?? this.visionConfig?.imageTokenId ?? this.imageTokenId;
|
|
1090
|
+
if (placeholderTokenId == null) {
|
|
1091
|
+
throw new Error(
|
|
1092
|
+
'Pipeline missing video/image token ID for video placeholder expansion.'
|
|
1093
|
+
);
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
const templateType = this.modelConfig?.chatTemplateType ?? 'gemma4';
|
|
1097
|
+
const chatOptions = this.modelConfig?.chatTemplateThinking === true ? { thinking: true } : undefined;
|
|
1098
|
+
const multimodalPrompt = formatChatMessages([
|
|
1099
|
+
{
|
|
1100
|
+
role: 'user',
|
|
1101
|
+
content: [
|
|
1102
|
+
{ type: 'video' },
|
|
1103
|
+
{ type: 'text', text: requestedPrompt },
|
|
1104
|
+
],
|
|
1105
|
+
},
|
|
1106
|
+
], templateType, chatOptions);
|
|
1107
|
+
const promptTokenIds = this.tokenizer.encode(multimodalPrompt);
|
|
1108
|
+
const videoTokenSpanLength = encodeResult.numTokens;
|
|
1109
|
+
|
|
1110
|
+
// Resolve BOV/EOV tokens (reuse image BOI/EOI if video-specific ones don't exist)
|
|
1111
|
+
const bovTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<|video|>', 'Gemma 4 BOV token');
|
|
1112
|
+
const eovTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<video|>', 'Gemma 4 EOV token');
|
|
1113
|
+
|
|
1114
|
+
const { inputIds: fullTokenIds, imageStartOffset: videoStartOffset } = expandImagePlaceholderTokenIds(
|
|
1115
|
+
promptTokenIds,
|
|
1116
|
+
placeholderTokenId,
|
|
1117
|
+
videoTokenSpanLength,
|
|
1118
|
+
{ boiTokenId: bovTokenId, eoiTokenId: eovTokenId }
|
|
1119
|
+
);
|
|
1120
|
+
|
|
1121
|
+
const padTokenId = this.tokenizer?.getSpecialTokens?.()?.pad;
|
|
1122
|
+
if (!Number.isFinite(padTokenId) || Math.floor(padTokenId) !== padTokenId || padTokenId < 0) {
|
|
1123
|
+
throw new Error(
|
|
1124
|
+
`[Pipeline] transcribeVideo: Gemma 4 multimodal prefill requires a tokenizer pad token ID, got ${padTokenId}.`
|
|
1125
|
+
);
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
// Step 3: Generate with embedding override at the video token offset
|
|
1129
|
+
const tokens = [];
|
|
1130
|
+
const maxGen = maxTokens ?? 512;
|
|
1131
|
+
const stopTokenIds = this.modelConfig.stopTokenIds;
|
|
1132
|
+
|
|
1133
|
+
try {
|
|
1134
|
+
const generation = await this.generator.generateTokenIds('', buildConservativeMultimodalGenerationOptions({
|
|
1135
|
+
inputIds: fullTokenIds,
|
|
1136
|
+
embeddingOverrides: {
|
|
1137
|
+
prefixLength: encodeResult.numTokens,
|
|
1138
|
+
offset: videoStartOffset,
|
|
1139
|
+
embeddings: encodeResult.features,
|
|
1140
|
+
},
|
|
1141
|
+
__internalEmbeddingInputSpan: {
|
|
1142
|
+
offset: videoStartOffset,
|
|
1143
|
+
length: encodeResult.numTokens,
|
|
1144
|
+
tokenId: padTokenId,
|
|
1145
|
+
},
|
|
1146
|
+
__internalMultimodalBidirectionalSpan: {
|
|
1147
|
+
offset: videoStartOffset,
|
|
1148
|
+
length: encodeResult.numTokens,
|
|
1149
|
+
},
|
|
1150
|
+
maxTokens: maxGen,
|
|
1151
|
+
temperature: 0,
|
|
1152
|
+
topK: 1,
|
|
1153
|
+
topP: 1,
|
|
1154
|
+
repetitionPenalty: 1,
|
|
1155
|
+
}));
|
|
1156
|
+
for (const token of generation.tokenIds ?? []) {
|
|
1157
|
+
if (Array.isArray(stopTokenIds) && stopTokenIds.includes(token)) break;
|
|
1158
|
+
tokens.push(token);
|
|
1159
|
+
}
|
|
1160
|
+
} finally {
|
|
1161
|
+
if (encodeResult.features) {
|
|
1162
|
+
releaseBuffer(encodeResult.features);
|
|
1163
|
+
}
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
const text = this.tokenizer.decode(tokens);
|
|
1167
|
+
return { text, tokens };
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
|
|
1171
|
+
// ==========================================================================
|
|
1172
|
+
// Audio: transcribeAudio
|
|
1173
|
+
// ==========================================================================
|
|
1174
|
+
|
|
1175
|
+
/**
|
|
1176
|
+
* Transcribe text from audio using the audio encoder and text decoder.
|
|
1177
|
+
*
|
|
1178
|
+
* @param {object} params
|
|
1179
|
+
* @param {Float32Array} params.audio Raw audio PCM (mono, 16kHz)
|
|
1180
|
+
* @param {string} [params.prompt] Custom transcription prompt
|
|
1181
|
+
* @param {number} [params.maxTokens] Max tokens to generate
|
|
1182
|
+
* @returns {Promise<{ text: string, tokens: number[] }>}
|
|
1183
|
+
*/
|
|
1184
|
+
async transcribeAudio({ audio, prompt, maxTokens }) {
|
|
1185
|
+
if (!this.audioCapable) {
|
|
1186
|
+
throw new Error(
|
|
1187
|
+
'Pipeline does not support audio transcription (no audio_token_id in manifest).'
|
|
1188
|
+
);
|
|
1189
|
+
}
|
|
1190
|
+
await this._ensureAudioWeightsLoaded();
|
|
1191
|
+
|
|
1192
|
+
this.reset();
|
|
1193
|
+
|
|
1194
|
+
// Lazy-load audio modules
|
|
1195
|
+
const { extractLogMelSpectrogram } = await import('./audio/mel.js');
|
|
1196
|
+
const { encodeAudio } = await import('./audio/index.js');
|
|
1197
|
+
|
|
1198
|
+
// Step 1: Extract mel spectrogram
|
|
1199
|
+
const { features: melFeatures, numFrames, nMels } = extractLogMelSpectrogram(audio);
|
|
1200
|
+
|
|
1201
|
+
// Step 2: Encode audio through conformer pipeline
|
|
1202
|
+
const encodeResult = await encodeAudio({
|
|
1203
|
+
melFeatures,
|
|
1204
|
+
numFrames,
|
|
1205
|
+
nMels,
|
|
1206
|
+
audioConfig: this.audioConfig,
|
|
1207
|
+
weights: this.audioWeights,
|
|
1208
|
+
});
|
|
1209
|
+
|
|
1210
|
+
// Step 3: Build the multimodal prompt with <|audio|> placeholder
|
|
1211
|
+
const requestedPrompt = prompt ?? 'Transcribe the audio.';
|
|
1212
|
+
const audioTokenId = this.audioConfig?.audioTokenId ?? this.audioTokenId;
|
|
1213
|
+
if (audioTokenId == null) {
|
|
1214
|
+
throw new Error(
|
|
1215
|
+
'Pipeline missing audio_token_id. Re-convert the model with audio token metadata.'
|
|
1216
|
+
);
|
|
1217
|
+
}
|
|
1218
|
+
const templateType = this.modelConfig?.chatTemplateType ?? 'gemma4';
|
|
1219
|
+
const chatOptions = this.modelConfig?.chatTemplateThinking === true ? { thinking: true } : undefined;
|
|
1220
|
+
const multimodalPrompt = formatChatMessages([
|
|
1221
|
+
{
|
|
1222
|
+
role: 'user',
|
|
1223
|
+
content: [
|
|
1224
|
+
{ type: 'audio' },
|
|
1225
|
+
{ type: 'text', text: requestedPrompt },
|
|
1226
|
+
],
|
|
1227
|
+
},
|
|
1228
|
+
], templateType, chatOptions);
|
|
1229
|
+
const promptTokenIds = this.tokenizer.encode(multimodalPrompt);
|
|
1230
|
+
const audioTokenSpanLength = encodeResult.numTokens;
|
|
1231
|
+
|
|
1232
|
+
// Resolve BOA/EOA tokens
|
|
1233
|
+
const boaTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<|audio|>', 'Gemma 4 BOA token');
|
|
1234
|
+
const eoaTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<audio|>', 'Gemma 4 EOA token');
|
|
1235
|
+
|
|
1236
|
+
// Expand single audio placeholder token into the full audio token span
|
|
1237
|
+
const { inputIds: fullTokenIds, imageStartOffset: audioStartOffset } = expandImagePlaceholderTokenIds(
|
|
1238
|
+
promptTokenIds,
|
|
1239
|
+
audioTokenId,
|
|
1240
|
+
audioTokenSpanLength,
|
|
1241
|
+
{ boiTokenId: boaTokenId, eoiTokenId: eoaTokenId }
|
|
1242
|
+
);
|
|
1243
|
+
|
|
1244
|
+
const padTokenId = this.tokenizer?.getSpecialTokens?.()?.pad;
|
|
1245
|
+
if (!Number.isFinite(padTokenId) || Math.floor(padTokenId) !== padTokenId || padTokenId < 0) {
|
|
1246
|
+
throw new Error(
|
|
1247
|
+
`[Pipeline] transcribeAudio: Gemma 4 multimodal prefill requires a tokenizer pad token ID, got ${padTokenId}.`
|
|
1248
|
+
);
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
// Step 4: Generate with embedding override at the audio token offset
|
|
1252
|
+
const tokens = [];
|
|
1253
|
+
const maxGen = maxTokens ?? 512;
|
|
1254
|
+
const stopTokenIds = this.modelConfig.stopTokenIds;
|
|
1255
|
+
|
|
1256
|
+
try {
|
|
1257
|
+
const generation = await this.generator.generateTokenIds('', buildConservativeMultimodalGenerationOptions({
|
|
1258
|
+
inputIds: fullTokenIds,
|
|
1259
|
+
embeddingOverrides: {
|
|
1260
|
+
prefixLength: encodeResult.numTokens,
|
|
1261
|
+
offset: audioStartOffset,
|
|
1262
|
+
embeddings: encodeResult.features,
|
|
1263
|
+
},
|
|
1264
|
+
__internalEmbeddingInputSpan: {
|
|
1265
|
+
offset: audioStartOffset,
|
|
1266
|
+
length: encodeResult.numTokens,
|
|
1267
|
+
tokenId: padTokenId,
|
|
1268
|
+
},
|
|
1269
|
+
__internalMultimodalBidirectionalSpan: {
|
|
1270
|
+
offset: audioStartOffset,
|
|
1271
|
+
length: encodeResult.numTokens,
|
|
1272
|
+
},
|
|
1273
|
+
maxTokens: maxGen,
|
|
1274
|
+
temperature: 0,
|
|
1275
|
+
topK: 1,
|
|
1276
|
+
topP: 1,
|
|
1277
|
+
repetitionPenalty: 1,
|
|
1278
|
+
}));
|
|
1279
|
+
for (const token of generation.tokenIds ?? []) {
|
|
1280
|
+
if (Array.isArray(stopTokenIds) && stopTokenIds.includes(token)) break;
|
|
1281
|
+
tokens.push(token);
|
|
1282
|
+
}
|
|
1283
|
+
} finally {
|
|
1284
|
+
if (encodeResult.features) {
|
|
1285
|
+
releaseBuffer(encodeResult.features);
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
const text = this.tokenizer.decode(tokens);
|
|
1290
|
+
return { text, tokens };
|
|
1291
|
+
}
|
|
1292
|
+
|
|
1293
|
+
|
|
1294
|
+
// ==========================================================================
|
|
1295
|
+
// Capability Detection
|
|
1296
|
+
// ==========================================================================
|
|
1297
|
+
|
|
1298
|
+
get capabilities() {
|
|
1299
|
+
const caps = ['generation'];
|
|
1300
|
+
if (typeof this.prefillWithEmbedding === 'function') caps.push('embedding');
|
|
1301
|
+
if (this.visionCapable) caps.push('multimodal');
|
|
1302
|
+
if (this.audioCapable) caps.push('audio');
|
|
1303
|
+
if (this.visionCapable) caps.push('video');
|
|
1304
|
+
return Object.freeze(caps);
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
|
|
492
1308
|
// Layer pipeline precedence (lowest to highest):
|
|
493
1309
|
// 1. execution-v1-produced pipeline (via runtimeInferencePatch.pipeline)
|
|
494
1310
|
// 2. model config pipeline (manifest inference.pipeline)
|
|
@@ -580,10 +1396,11 @@ export class InferencePipeline extends PipelineState {
|
|
|
580
1396
|
if (!Array.isArray(prompts)) {
|
|
581
1397
|
throw new Error('embedBatch expects an array of prompts');
|
|
582
1398
|
}
|
|
1399
|
+
const batchOptions = { ...options, __skipStateSnapshot: true };
|
|
583
1400
|
const outputs = [];
|
|
584
1401
|
for (const prompt of prompts) {
|
|
585
|
-
outputs.push(await this.embed(prompt,
|
|
586
|
-
this.
|
|
1402
|
+
outputs.push(await this.embed(prompt, batchOptions));
|
|
1403
|
+
this.resetForBatch();
|
|
587
1404
|
}
|
|
588
1405
|
return outputs;
|
|
589
1406
|
}
|
|
@@ -629,6 +1446,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
629
1446
|
|
|
630
1447
|
getStats() {
|
|
631
1448
|
const stats = { ...this.stats };
|
|
1449
|
+
stats.batching ??= { ...this.batchingStats };
|
|
632
1450
|
if (this.executionPlanState) {
|
|
633
1451
|
const activeExecutionPlan = resolveActiveExecutionPlan(this);
|
|
634
1452
|
stats.executionPlan ??= {
|
|
@@ -734,6 +1552,10 @@ export class InferencePipeline extends PipelineState {
|
|
|
734
1552
|
this.emulation = null;
|
|
735
1553
|
this.decodeRing?.release();
|
|
736
1554
|
this.kvCache?.clear();
|
|
1555
|
+
destroyPleRuntimeCache(this.weights.get('per_layer_inputs'));
|
|
1556
|
+
destroyPleBufferCache(this.pleCache);
|
|
1557
|
+
this.pleCache = null;
|
|
1558
|
+
this.plePrefetchPending = null;
|
|
737
1559
|
this.weights.clear();
|
|
738
1560
|
this.expertWeights.clear();
|
|
739
1561
|
this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
|
|
@@ -744,6 +1566,10 @@ export class InferencePipeline extends PipelineState {
|
|
|
744
1566
|
this.finitenessBuffer.destroy();
|
|
745
1567
|
this.finitenessBuffer = null;
|
|
746
1568
|
}
|
|
1569
|
+
if (this.sampleReadbackBuffer) {
|
|
1570
|
+
this.sampleReadbackBuffer.destroy();
|
|
1571
|
+
this.sampleReadbackBuffer = null;
|
|
1572
|
+
}
|
|
747
1573
|
this.isLoaded = false;
|
|
748
1574
|
this.currentSeqLen = 0;
|
|
749
1575
|
restorePipelineContexts(this);
|
|
@@ -776,6 +1602,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
776
1602
|
this.stats.decodeTimeMs = 0;
|
|
777
1603
|
this.stats.gpuTimePrefillMs = undefined;
|
|
778
1604
|
this.stats.gpuTimeDecodeMs = undefined;
|
|
1605
|
+
this.stats.prefillProfileSteps = [];
|
|
779
1606
|
this.stats.decodeProfileSteps = [];
|
|
780
1607
|
this.stats.executionPlan = null;
|
|
781
1608
|
this.stats.kernelPathId = null;
|
|
@@ -783,16 +1610,37 @@ export class InferencePipeline extends PipelineState {
|
|
|
783
1610
|
this.stats.attentionInputs = [];
|
|
784
1611
|
}
|
|
785
1612
|
|
|
1613
|
+
/**
|
|
1614
|
+
* Lightweight reset for batch embedding: clears KV cache and sequence state
|
|
1615
|
+
* without resetting stats or debug flags (they are overwritten on each prefill).
|
|
1616
|
+
*/
|
|
1617
|
+
resetForBatch() {
|
|
1618
|
+
this.kvCache?.clear();
|
|
1619
|
+
this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
|
|
1620
|
+
this.currentSeqLen = 0;
|
|
1621
|
+
this.decodeStepCount = 0;
|
|
1622
|
+
this.decodeBuffers?.resetPingPong();
|
|
1623
|
+
this.decodeRing?.reset();
|
|
1624
|
+
}
|
|
1625
|
+
|
|
786
1626
|
|
|
787
1627
|
releaseGPUResources() {
|
|
788
1628
|
this.decodeBuffers?.release();
|
|
789
1629
|
this.decodeRing?.release();
|
|
790
1630
|
destroyMoERouter(this.moeRouter);
|
|
791
1631
|
this.moeRouter = null;
|
|
1632
|
+
destroyPleRuntimeCache(this.weights.get('per_layer_inputs'));
|
|
1633
|
+
destroyPleBufferCache(this.pleCache);
|
|
1634
|
+
this.pleCache = null;
|
|
1635
|
+
this.plePrefetchPending = null;
|
|
792
1636
|
if (this.finitenessBuffer) {
|
|
793
1637
|
this.finitenessBuffer.destroy();
|
|
794
1638
|
this.finitenessBuffer = null;
|
|
795
1639
|
}
|
|
1640
|
+
if (this.sampleReadbackBuffer) {
|
|
1641
|
+
this.sampleReadbackBuffer.destroy();
|
|
1642
|
+
this.sampleReadbackBuffer = null;
|
|
1643
|
+
}
|
|
796
1644
|
}
|
|
797
1645
|
}
|
|
798
1646
|
|
|
@@ -806,6 +1654,7 @@ async function createTransformerPipeline(manifest, contexts = {}) {
|
|
|
806
1654
|
}
|
|
807
1655
|
|
|
808
1656
|
registerPipeline('transformer', createTransformerPipeline);
|
|
1657
|
+
registerPipeline('gemma4', createTransformerPipeline);
|
|
809
1658
|
|
|
810
1659
|
export class EmbeddingPipeline extends InferencePipeline {
|
|
811
1660
|
async *generate() {
|