doppler-gpu 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { RDRRManifest } from '../../formats/rdrr/index.js';
|
|
2
|
+
import type { InferencePipeline } from '../../inference/pipelines/text.js';
|
|
3
|
+
import type { LoRAManifest } from './types.js';
|
|
4
|
+
|
|
5
|
+
export declare function loadLoRAAdapterForPipeline(
|
|
6
|
+
pipeline: InferencePipeline | null | undefined,
|
|
7
|
+
adapter: LoRAManifest | RDRRManifest | string
|
|
8
|
+
): Promise<void>;
|
|
9
|
+
|
|
10
|
+
export declare function activateLoRAFromTrainingOutputForPipeline(
|
|
11
|
+
pipeline: InferencePipeline | null | undefined,
|
|
12
|
+
trainingOutput:
|
|
13
|
+
| string
|
|
14
|
+
| {
|
|
15
|
+
adapter?: LoRAManifest | RDRRManifest | string;
|
|
16
|
+
adapterManifest?: LoRAManifest | RDRRManifest;
|
|
17
|
+
adapterManifestJson?: string;
|
|
18
|
+
adapterManifestUrl?: string;
|
|
19
|
+
adapterManifestPath?: string;
|
|
20
|
+
}
|
|
21
|
+
| null
|
|
22
|
+
| undefined
|
|
23
|
+
): Promise<{
|
|
24
|
+
activated: boolean;
|
|
25
|
+
adapterName: string | null;
|
|
26
|
+
source: string | null;
|
|
27
|
+
reason: string | null;
|
|
28
|
+
}>;
|
|
29
|
+
|
|
30
|
+
export declare function unloadLoRAAdapterForPipeline(
|
|
31
|
+
pipeline: InferencePipeline | null | undefined
|
|
32
|
+
): Promise<void>;
|
|
33
|
+
|
|
34
|
+
export declare function getActiveLoRAForPipeline(
|
|
35
|
+
pipeline: InferencePipeline | null | undefined
|
|
36
|
+
): string | null;
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { log } from '../../debug/index.js';
|
|
2
|
+
import { getDopplerLoader } from '../../loader/doppler-loader.js';
|
|
3
|
+
import { fetchArrayBuffer, readOPFSFile, writeOPFSFile } from './storage.js';
|
|
4
|
+
|
|
5
|
+
let loraModulePromise = null;
|
|
6
|
+
|
|
7
|
+
async function getExperimentalLoRAModule() {
|
|
8
|
+
loraModulePromise ??= import('../../experimental/adapters/lora-loader.js');
|
|
9
|
+
return loraModulePromise;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function createLoRALoadOptions() {
|
|
13
|
+
return {
|
|
14
|
+
readOPFS: readOPFSFile,
|
|
15
|
+
writeOPFS: writeOPFSFile,
|
|
16
|
+
fetchUrl: fetchArrayBuffer,
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function getActiveLoRAName(pipeline) {
|
|
21
|
+
const active = pipeline?.getActiveLoRA?.() || null;
|
|
22
|
+
return active ? active.name : null;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export async function loadLoRAAdapterForPipeline(pipeline, adapter) {
|
|
26
|
+
if (!pipeline) {
|
|
27
|
+
throw new Error('No model loaded. Call load() first.');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const options = createLoRALoadOptions();
|
|
31
|
+
let lora;
|
|
32
|
+
if (typeof adapter === 'string') {
|
|
33
|
+
const { loadLoRAFromUrl } = await getExperimentalLoRAModule();
|
|
34
|
+
lora = await loadLoRAFromUrl(adapter, options);
|
|
35
|
+
} else if (adapter?.adapterType === 'lora' || adapter?.modelType === 'lora') {
|
|
36
|
+
const loader = pipeline.dopplerLoader || getDopplerLoader();
|
|
37
|
+
await loader.init();
|
|
38
|
+
lora = await loader.loadLoRAWeights(adapter);
|
|
39
|
+
} else {
|
|
40
|
+
const { loadLoRAFromManifest } = await getExperimentalLoRAModule();
|
|
41
|
+
lora = await loadLoRAFromManifest(adapter, options);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
pipeline.setLoRAAdapter(lora);
|
|
45
|
+
log.info('doppler', `LoRA adapter loaded: ${lora.name}`);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export async function activateLoRAFromTrainingOutputForPipeline(pipeline, trainingOutput) {
|
|
49
|
+
if (!pipeline) {
|
|
50
|
+
return {
|
|
51
|
+
activated: false,
|
|
52
|
+
adapterName: null,
|
|
53
|
+
source: null,
|
|
54
|
+
reason: 'no_model_loaded',
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const output = trainingOutput && typeof trainingOutput === 'object'
|
|
59
|
+
? trainingOutput
|
|
60
|
+
: null;
|
|
61
|
+
if (!output && typeof trainingOutput !== 'string') {
|
|
62
|
+
return {
|
|
63
|
+
activated: false,
|
|
64
|
+
adapterName: getActiveLoRAName(pipeline),
|
|
65
|
+
source: null,
|
|
66
|
+
reason: 'no_adapter_candidate',
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (typeof trainingOutput === 'string') {
|
|
71
|
+
await loadLoRAAdapterForPipeline(pipeline, trainingOutput);
|
|
72
|
+
return {
|
|
73
|
+
activated: true,
|
|
74
|
+
adapterName: getActiveLoRAName(pipeline),
|
|
75
|
+
source: 'adapter-string',
|
|
76
|
+
reason: null,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
if (output.adapterManifest && typeof output.adapterManifest === 'object') {
|
|
81
|
+
await loadLoRAAdapterForPipeline(pipeline, output.adapterManifest);
|
|
82
|
+
return {
|
|
83
|
+
activated: true,
|
|
84
|
+
adapterName: getActiveLoRAName(pipeline),
|
|
85
|
+
source: 'adapterManifest',
|
|
86
|
+
reason: null,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
if (typeof output.adapterManifestJson === 'string' && output.adapterManifestJson.trim()) {
|
|
91
|
+
const manifest = JSON.parse(output.adapterManifestJson);
|
|
92
|
+
await loadLoRAAdapterForPipeline(pipeline, manifest);
|
|
93
|
+
return {
|
|
94
|
+
activated: true,
|
|
95
|
+
adapterName: getActiveLoRAName(pipeline),
|
|
96
|
+
source: 'adapterManifestJson',
|
|
97
|
+
reason: null,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
if (typeof output.adapterManifestUrl === 'string' && output.adapterManifestUrl.trim()) {
|
|
102
|
+
await loadLoRAAdapterForPipeline(pipeline, output.adapterManifestUrl.trim());
|
|
103
|
+
return {
|
|
104
|
+
activated: true,
|
|
105
|
+
adapterName: getActiveLoRAName(pipeline),
|
|
106
|
+
source: 'adapterManifestUrl',
|
|
107
|
+
reason: null,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
if (typeof output.adapterManifestPath === 'string' && output.adapterManifestPath.trim()) {
|
|
112
|
+
const path = output.adapterManifestPath.trim();
|
|
113
|
+
if (path.startsWith('http://') || path.startsWith('https://')) {
|
|
114
|
+
await loadLoRAAdapterForPipeline(pipeline, path);
|
|
115
|
+
return {
|
|
116
|
+
activated: true,
|
|
117
|
+
adapterName: getActiveLoRAName(pipeline),
|
|
118
|
+
source: 'adapterManifestPath:url',
|
|
119
|
+
reason: null,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const isNode = typeof process !== 'undefined' && !!process.versions?.node;
|
|
124
|
+
if (!isNode) {
|
|
125
|
+
throw new Error('adapterManifestPath local files require Node runtime.');
|
|
126
|
+
}
|
|
127
|
+
const { readFile } = await import('node:fs/promises');
|
|
128
|
+
const raw = await readFile(path, 'utf8');
|
|
129
|
+
const manifest = JSON.parse(raw);
|
|
130
|
+
await loadLoRAAdapterForPipeline(pipeline, manifest);
|
|
131
|
+
return {
|
|
132
|
+
activated: true,
|
|
133
|
+
adapterName: getActiveLoRAName(pipeline),
|
|
134
|
+
source: 'adapterManifestPath:file',
|
|
135
|
+
reason: null,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if (output.adapter != null) {
|
|
140
|
+
await loadLoRAAdapterForPipeline(pipeline, output.adapter);
|
|
141
|
+
return {
|
|
142
|
+
activated: true,
|
|
143
|
+
adapterName: getActiveLoRAName(pipeline),
|
|
144
|
+
source: 'adapter',
|
|
145
|
+
reason: null,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return {
|
|
150
|
+
activated: false,
|
|
151
|
+
adapterName: getActiveLoRAName(pipeline),
|
|
152
|
+
source: null,
|
|
153
|
+
reason: 'no_adapter_candidate',
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export async function unloadLoRAAdapterForPipeline(pipeline) {
|
|
158
|
+
if (!pipeline) return;
|
|
159
|
+
pipeline.setLoRAAdapter(null);
|
|
160
|
+
log.info('doppler', 'LoRA adapter unloaded');
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export function getActiveLoRAForPipeline(pipeline) {
|
|
164
|
+
return getActiveLoRAName(pipeline);
|
|
165
|
+
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { InferencePipeline } from '../../inference/pipelines/text.js';
|
|
2
2
|
import type { RDRRManifest } from '../../formats/rdrr/index.js';
|
|
3
|
-
import type { LoRAManifest } from '../../adapters/lora-loader.js';
|
|
4
3
|
import type {
|
|
5
4
|
TextModelConfig,
|
|
6
5
|
LoadProgressEvent,
|
|
6
|
+
LoRAManifest,
|
|
7
7
|
} from './types.js';
|
|
8
8
|
|
|
9
9
|
export declare function getPipeline(): InferencePipeline | null;
|
|
@@ -13,8 +13,6 @@ import { requestPersistence, getStorageReport } from '../../storage/quota.js';
|
|
|
13
13
|
import { initDevice, getKernelCapabilities, getDeviceLimits, destroyDevice, getDevice } from '../../gpu/device.js';
|
|
14
14
|
import { prepareKernelRuntime } from '../../gpu/kernel-runtime.js';
|
|
15
15
|
import { createPipeline } from '../../inference/pipelines/text.js';
|
|
16
|
-
import { isBridgeAvailable, createBridgeClient } from '../../bridge/index.js';
|
|
17
|
-
import { loadLoRAFromManifest, loadLoRAFromUrl } from '../../adapters/lora-loader.js';
|
|
18
16
|
import { getDopplerLoader } from '../../loader/doppler-loader.js';
|
|
19
17
|
import { log } from '../../debug/index.js';
|
|
20
18
|
import { DopplerCapabilities } from './types.js';
|
|
@@ -24,11 +22,24 @@ import { getRuntimeConfig } from '../../config/runtime.js';
|
|
|
24
22
|
import {
|
|
25
23
|
buildSourceArtifactFingerprint,
|
|
26
24
|
createStoredSourceArtifactContext,
|
|
25
|
+
synthesizeStoredSourceArtifactManifest,
|
|
27
26
|
verifyStoredSourceArtifact,
|
|
28
27
|
} from '../../storage/source-artifact-store.js';
|
|
29
28
|
|
|
30
29
|
let pipeline = null;
|
|
31
30
|
let currentModelId = null;
|
|
31
|
+
let bridgeModulePromise = null;
|
|
32
|
+
let loraModulePromise = null;
|
|
33
|
+
|
|
34
|
+
async function getExperimentalBridgeModule() {
|
|
35
|
+
bridgeModulePromise ??= import('../../experimental/bridge/index.js');
|
|
36
|
+
return bridgeModulePromise;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function getExperimentalLoRAModule() {
|
|
40
|
+
loraModulePromise ??= import('../../experimental/adapters/lora-loader.js');
|
|
41
|
+
return loraModulePromise;
|
|
42
|
+
}
|
|
32
43
|
|
|
33
44
|
function manifestsDiffer(localManifest, remoteManifest) {
|
|
34
45
|
if (!localManifest || !remoteManifest) return true;
|
|
@@ -264,12 +275,16 @@ export async function loadModel(modelId, modelUrl = null, onProgress = null, loc
|
|
|
264
275
|
try {
|
|
265
276
|
log.info('DopplerProvider', `Loading model: ${modelId}`);
|
|
266
277
|
|
|
267
|
-
let
|
|
278
|
+
let runtimeModel = null;
|
|
268
279
|
let useBridge = false;
|
|
269
280
|
let bridgeStorageContext = null;
|
|
270
281
|
let bridgeSourceMode = false;
|
|
271
282
|
|
|
272
|
-
if (localPath
|
|
283
|
+
if (localPath) {
|
|
284
|
+
const { isBridgeAvailable, createBridgeClient } = await getExperimentalBridgeModule();
|
|
285
|
+
if (!isBridgeAvailable()) {
|
|
286
|
+
throw new Error('Local path loading requires the experimental bridge surface, but no bridge is available.');
|
|
287
|
+
}
|
|
273
288
|
log.info('DopplerProvider', `Using Native Bridge for local path: ${localPath}`);
|
|
274
289
|
useBridge = true;
|
|
275
290
|
|
|
@@ -285,14 +300,14 @@ export async function loadModel(modelId, modelUrl = null, onProgress = null, loc
|
|
|
285
300
|
try {
|
|
286
301
|
const manifestBytes = await bridgeClient.read(manifestPath, 0, HEADER_READ_SIZE);
|
|
287
302
|
const manifestJson = new TextDecoder().decode(manifestBytes);
|
|
288
|
-
|
|
289
|
-
log.info('DopplerProvider', `Loaded manifest via bridge: ${
|
|
303
|
+
runtimeModel = parseManifest(manifestJson);
|
|
304
|
+
log.info('DopplerProvider', `Loaded manifest via bridge: ${runtimeModel.modelId}`);
|
|
290
305
|
if (onProgress) onProgress({ stage: 'manifest', message: 'Manifest loaded via bridge' });
|
|
291
306
|
const persistedSourceBundle = await resolveBridgeSourceRuntimeBundle({
|
|
292
307
|
bridgeClient,
|
|
293
308
|
localPath,
|
|
294
309
|
modelId,
|
|
295
|
-
|
|
310
|
+
model: runtimeModel,
|
|
296
311
|
verifyHashes: true,
|
|
297
312
|
onProgress: (progress) => onProgress?.(progress),
|
|
298
313
|
});
|
|
@@ -302,7 +317,7 @@ export async function loadModel(modelId, modelUrl = null, onProgress = null, loc
|
|
|
302
317
|
if (onProgress) {
|
|
303
318
|
onProgress({
|
|
304
319
|
stage: 'manifest',
|
|
305
|
-
message: `Direct-source
|
|
320
|
+
message: `Direct-source runtime model ready (${persistedSourceBundle.sourceKind} artifact mode)`,
|
|
306
321
|
});
|
|
307
322
|
}
|
|
308
323
|
}
|
|
@@ -321,13 +336,13 @@ export async function loadModel(modelId, modelUrl = null, onProgress = null, loc
|
|
|
321
336
|
if (!sourceBundle) {
|
|
322
337
|
throw manifestError;
|
|
323
338
|
}
|
|
324
|
-
|
|
339
|
+
runtimeModel = sourceBundle.model ?? sourceBundle.manifest;
|
|
325
340
|
bridgeStorageContext = sourceBundle.storageContext;
|
|
326
341
|
bridgeSourceMode = true;
|
|
327
342
|
if (onProgress) {
|
|
328
343
|
onProgress({
|
|
329
344
|
stage: 'manifest',
|
|
330
|
-
message: `
|
|
345
|
+
message: `Runtime model ready (${sourceBundle.sourceKind} direct-source mode)`,
|
|
331
346
|
});
|
|
332
347
|
}
|
|
333
348
|
}
|
|
@@ -343,16 +358,16 @@ export async function loadModel(modelId, modelUrl = null, onProgress = null, loc
|
|
|
343
358
|
|
|
344
359
|
try {
|
|
345
360
|
const manifestJson = await loadManifestFromStore();
|
|
346
|
-
|
|
361
|
+
runtimeModel = parseManifest(manifestJson);
|
|
347
362
|
} catch {
|
|
348
|
-
|
|
363
|
+
runtimeModel = null;
|
|
349
364
|
}
|
|
350
365
|
|
|
351
366
|
let integrity = { valid: false, missingShards: [] };
|
|
352
|
-
if (
|
|
353
|
-
const sourceArtifactFingerprint = buildSourceArtifactFingerprint(
|
|
367
|
+
if (runtimeModel) {
|
|
368
|
+
const sourceArtifactFingerprint = buildSourceArtifactFingerprint(runtimeModel);
|
|
354
369
|
if (sourceArtifactFingerprint) {
|
|
355
|
-
const sourceIntegrity = await verifyStoredSourceArtifact(
|
|
370
|
+
const sourceIntegrity = await verifyStoredSourceArtifact(runtimeModel, { checkHashes: false }).catch(() => ({
|
|
356
371
|
valid: false,
|
|
357
372
|
missingFiles: [],
|
|
358
373
|
}));
|
|
@@ -368,8 +383,8 @@ export async function loadModel(modelId, modelUrl = null, onProgress = null, loc
|
|
|
368
383
|
}
|
|
369
384
|
}
|
|
370
385
|
|
|
371
|
-
if (integrity.valid &&
|
|
372
|
-
await verifyExplicitModelUrlMatch(
|
|
386
|
+
if (integrity.valid && runtimeModel && modelUrl) {
|
|
387
|
+
await verifyExplicitModelUrlMatch(runtimeModel, modelUrl);
|
|
373
388
|
}
|
|
374
389
|
|
|
375
390
|
if (!integrity.valid && modelUrl) {
|
|
@@ -382,17 +397,26 @@ export async function loadModel(modelId, modelUrl = null, onProgress = null, loc
|
|
|
382
397
|
throw new Error(`Model ${modelId} not found and no URL provided`);
|
|
383
398
|
}
|
|
384
399
|
|
|
385
|
-
|
|
400
|
+
runtimeModel = getManifest();
|
|
386
401
|
}
|
|
387
402
|
|
|
388
|
-
if (!
|
|
403
|
+
if (!runtimeModel) {
|
|
389
404
|
throw new Error('Failed to load model manifest');
|
|
390
405
|
}
|
|
391
406
|
|
|
407
|
+
const synthesizedRuntimeModel = synthesizeStoredSourceArtifactManifest(runtimeModel);
|
|
408
|
+
if (synthesizedRuntimeModel.changed) {
|
|
409
|
+
log.info(
|
|
410
|
+
'DopplerProvider',
|
|
411
|
+
`Enabled stored-shard source runtime for "${runtimeModel.modelId ?? modelId}" warm loads`
|
|
412
|
+
);
|
|
413
|
+
runtimeModel = synthesizedRuntimeModel.manifest;
|
|
414
|
+
}
|
|
415
|
+
|
|
392
416
|
try {
|
|
393
|
-
const mc = extractTextModelConfig(
|
|
417
|
+
const mc = extractTextModelConfig(runtimeModel);
|
|
394
418
|
const kvBytes = mc.numLayers * mc.maxSeqLen * mc.numKVHeads * mc.headDim * 4 * 2;
|
|
395
|
-
const weightBytes = estimateDequantizedWeightsBytes(
|
|
419
|
+
const weightBytes = estimateDequantizedWeightsBytes(runtimeModel);
|
|
396
420
|
const estimate = {
|
|
397
421
|
weightsBytes: weightBytes,
|
|
398
422
|
kvCacheBytes: kvBytes,
|
|
@@ -414,13 +438,13 @@ export async function loadModel(modelId, modelUrl = null, onProgress = null, loc
|
|
|
414
438
|
log.warn('DopplerProvider', 'Failed to estimate GPU memory', e);
|
|
415
439
|
}
|
|
416
440
|
|
|
417
|
-
if (
|
|
441
|
+
if (runtimeModel.totalSize > DopplerCapabilities.MAX_MODEL_SIZE) {
|
|
418
442
|
throw new Error(
|
|
419
|
-
`Model size ${
|
|
443
|
+
`Model size ${runtimeModel.totalSize} exceeds max ${DopplerCapabilities.MAX_MODEL_SIZE}`
|
|
420
444
|
);
|
|
421
445
|
}
|
|
422
446
|
|
|
423
|
-
if (!DopplerCapabilities.IS_UNIFIED_MEMORY && !
|
|
447
|
+
if (!DopplerCapabilities.IS_UNIFIED_MEMORY && !runtimeModel.moeConfig) {
|
|
424
448
|
log.warn('DopplerProvider', 'Dense model on discrete GPU - performance will be limited');
|
|
425
449
|
}
|
|
426
450
|
|
|
@@ -436,7 +460,7 @@ export async function loadModel(modelId, modelUrl = null, onProgress = null, loc
|
|
|
436
460
|
&& typeof setTimeout !== 'undefined'
|
|
437
461
|
) {
|
|
438
462
|
DopplerCapabilities.kernelsTuned = true;
|
|
439
|
-
const tuneConfig = extractTextModelConfig(
|
|
463
|
+
const tuneConfig = extractTextModelConfig(runtimeModel);
|
|
440
464
|
setTimeout(() => {
|
|
441
465
|
prepareKernelRuntime({
|
|
442
466
|
prewarm: false,
|
|
@@ -458,8 +482,8 @@ export async function loadModel(modelId, modelUrl = null, onProgress = null, loc
|
|
|
458
482
|
const memCaps = await getMemoryCapabilities();
|
|
459
483
|
|
|
460
484
|
let storageContext = bridgeStorageContext;
|
|
461
|
-
if (!storageContext && buildSourceArtifactFingerprint(
|
|
462
|
-
storageContext = createStoredSourceArtifactContext(
|
|
485
|
+
if (!storageContext && buildSourceArtifactFingerprint(runtimeModel)) {
|
|
486
|
+
storageContext = createStoredSourceArtifactContext(runtimeModel, { verifyHashes: true });
|
|
463
487
|
}
|
|
464
488
|
if (!storageContext && useBridge && DopplerCapabilities.bridgeClient && DopplerCapabilities.localPath) {
|
|
465
489
|
const bridgeClient = DopplerCapabilities.bridgeClient;
|
|
@@ -467,9 +491,9 @@ export async function loadModel(modelId, modelUrl = null, onProgress = null, loc
|
|
|
467
491
|
? DopplerCapabilities.localPath
|
|
468
492
|
: `${DopplerCapabilities.localPath}/`;
|
|
469
493
|
|
|
470
|
-
const
|
|
494
|
+
const runtimeModelRef = runtimeModel;
|
|
471
495
|
const resolveShard = (idx) => {
|
|
472
|
-
const shardInfo =
|
|
496
|
+
const shardInfo = runtimeModelRef.shards[idx];
|
|
473
497
|
if (!shardInfo) throw new Error(`Invalid shard index: ${idx}`);
|
|
474
498
|
return {
|
|
475
499
|
shardInfo,
|
|
@@ -546,7 +570,7 @@ export async function loadModel(modelId, modelUrl = null, onProgress = null, loc
|
|
|
546
570
|
pipelineContexts.storage = storageContext;
|
|
547
571
|
}
|
|
548
572
|
|
|
549
|
-
pipeline = await createPipeline(
|
|
573
|
+
pipeline = await createPipeline(runtimeModel, pipelineContexts);
|
|
550
574
|
|
|
551
575
|
currentModelId = modelId;
|
|
552
576
|
DopplerCapabilities.currentModelId = modelId;
|
|
@@ -583,12 +607,14 @@ export async function loadLoRAAdapter(adapter) {
|
|
|
583
607
|
|
|
584
608
|
let lora;
|
|
585
609
|
if (typeof adapter === 'string') {
|
|
610
|
+
const { loadLoRAFromUrl } = await getExperimentalLoRAModule();
|
|
586
611
|
lora = await loadLoRAFromUrl(adapter, options);
|
|
587
612
|
} else if (adapter.adapterType === 'lora' || adapter.modelType === 'lora') {
|
|
588
613
|
const loader = pipeline.dopplerLoader || getDopplerLoader();
|
|
589
614
|
await loader.init();
|
|
590
615
|
lora = await loader.loadLoRAWeights(adapter);
|
|
591
616
|
} else {
|
|
617
|
+
const { loadLoRAFromManifest } = await getExperimentalLoRAModule();
|
|
592
618
|
lora = await loadLoRAFromManifest(adapter, options);
|
|
593
619
|
}
|
|
594
620
|
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { InferencePipeline, KVCacheSnapshot } from '../../inference/pipelines/text.js';
|
|
2
|
+
import type { ChatMessage } from '../../inference/pipelines/text/chat-format.js';
|
|
3
|
+
import type { GenerateOptions } from '../../generation/index.js';
|
|
4
|
+
import type { RDRRManifest } from '../../formats/rdrr/index.js';
|
|
5
|
+
import type { LogitsStepResult, PrefillResult } from '../../inference/pipelines/text/types.d.ts';
|
|
6
|
+
import type { LoRAManifest } from './types.js';
|
|
7
|
+
|
|
8
|
+
export type DopplerGenerateOptions = Omit<GenerateOptions, 'stopTokens'>;
|
|
9
|
+
|
|
10
|
+
export interface DopplerChatResponse {
|
|
11
|
+
content: string;
|
|
12
|
+
usage: {
|
|
13
|
+
promptTokens: number;
|
|
14
|
+
completionTokens: number;
|
|
15
|
+
totalTokens: number;
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface DopplerModelHandle {
|
|
20
|
+
generate(prompt: string, options?: DopplerGenerateOptions): AsyncGenerator<string, void, void>;
|
|
21
|
+
generateText(prompt: string, options?: DopplerGenerateOptions): Promise<string>;
|
|
22
|
+
chat(messages: ChatMessage[], options?: DopplerGenerateOptions): AsyncGenerator<string, void, void>;
|
|
23
|
+
chatText(messages: ChatMessage[], options?: DopplerGenerateOptions): Promise<DopplerChatResponse>;
|
|
24
|
+
loadLoRA(adapter: LoRAManifest | RDRRManifest | string): Promise<void>;
|
|
25
|
+
activateLoRAFromTrainingOutput(
|
|
26
|
+
trainingOutput:
|
|
27
|
+
| string
|
|
28
|
+
| {
|
|
29
|
+
adapter?: LoRAManifest | RDRRManifest | string;
|
|
30
|
+
adapterManifest?: LoRAManifest | RDRRManifest;
|
|
31
|
+
adapterManifestJson?: string;
|
|
32
|
+
adapterManifestUrl?: string;
|
|
33
|
+
adapterManifestPath?: string;
|
|
34
|
+
}
|
|
35
|
+
| null
|
|
36
|
+
| undefined
|
|
37
|
+
): Promise<{
|
|
38
|
+
activated: boolean;
|
|
39
|
+
adapterName: string | null;
|
|
40
|
+
source: string | null;
|
|
41
|
+
reason: string | null;
|
|
42
|
+
}>;
|
|
43
|
+
unloadLoRA(): Promise<void>;
|
|
44
|
+
unload(): Promise<void>;
|
|
45
|
+
readonly activeLoRA: string | null;
|
|
46
|
+
readonly loaded: boolean;
|
|
47
|
+
readonly modelId: string;
|
|
48
|
+
readonly manifest: unknown;
|
|
49
|
+
readonly deviceInfo: Record<string, unknown> | null;
|
|
50
|
+
readonly advanced: {
|
|
51
|
+
prefillKV(prompt: string, options?: DopplerGenerateOptions): Promise<KVCacheSnapshot>;
|
|
52
|
+
prefillWithLogits(
|
|
53
|
+
prompt: string | ChatMessage[] | { messages: ChatMessage[] },
|
|
54
|
+
options?: DopplerGenerateOptions
|
|
55
|
+
): Promise<PrefillResult>;
|
|
56
|
+
decodeStepLogits(currentIds: number[], options?: DopplerGenerateOptions): Promise<LogitsStepResult>;
|
|
57
|
+
generateWithPrefixKV(
|
|
58
|
+
prefix: KVCacheSnapshot,
|
|
59
|
+
prompt: string,
|
|
60
|
+
options?: DopplerGenerateOptions
|
|
61
|
+
): AsyncGenerator<string, void, void>;
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export declare function assertSupportedGenerationOptions(options?: Record<string, unknown>): void;
|
|
66
|
+
|
|
67
|
+
export declare function createModelHandle(
|
|
68
|
+
pipeline: InferencePipeline,
|
|
69
|
+
resolved: {
|
|
70
|
+
modelId: string;
|
|
71
|
+
}
|
|
72
|
+
): DopplerModelHandle;
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import { getKernelCapabilities } from '../../gpu/device.js';
|
|
2
|
+
import { formatChatMessages } from '../../inference/pipelines/text/chat-format.js';
|
|
3
|
+
import {
|
|
4
|
+
activateLoRAFromTrainingOutputForPipeline,
|
|
5
|
+
getActiveLoRAForPipeline,
|
|
6
|
+
loadLoRAAdapterForPipeline,
|
|
7
|
+
unloadLoRAAdapterForPipeline,
|
|
8
|
+
} from './lora.js';
|
|
9
|
+
|
|
10
|
+
export function assertSupportedGenerationOptions(options = {}) {
|
|
11
|
+
if (Array.isArray(options?.stopTokens) && options.stopTokens.length > 0) {
|
|
12
|
+
throw new Error(
|
|
13
|
+
'Doppler generate options do not support stopTokens on this surface. ' +
|
|
14
|
+
'Use stopSequences instead.'
|
|
15
|
+
);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function countTokens(pipeline, text) {
|
|
20
|
+
if (!text || typeof text !== 'string') return 0;
|
|
21
|
+
try {
|
|
22
|
+
return pipeline?.tokenizer?.encode(text)?.length ?? 0;
|
|
23
|
+
} catch {
|
|
24
|
+
return 0;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function resolveChatPromptForUsage(pipeline, messages) {
|
|
29
|
+
const templateType = pipeline?.manifest?.inference?.chatTemplate?.enabled === false
|
|
30
|
+
? null
|
|
31
|
+
: (pipeline?.manifest?.inference?.chatTemplate?.type ?? null);
|
|
32
|
+
try {
|
|
33
|
+
return formatChatMessages(messages, templateType);
|
|
34
|
+
} catch {
|
|
35
|
+
return messages.map((message) => String(message?.content ?? '')).join('\n');
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function collectText(iterable) {
|
|
40
|
+
let output = '';
|
|
41
|
+
for await (const token of iterable) {
|
|
42
|
+
output += token;
|
|
43
|
+
}
|
|
44
|
+
return output;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function createModelHandle(pipeline, resolved) {
|
|
48
|
+
return {
|
|
49
|
+
generate(prompt, options = {}) {
|
|
50
|
+
assertSupportedGenerationOptions(options);
|
|
51
|
+
return pipeline.generate(prompt, options);
|
|
52
|
+
},
|
|
53
|
+
async generateText(prompt, options = {}) {
|
|
54
|
+
assertSupportedGenerationOptions(options);
|
|
55
|
+
return collectText(pipeline.generate(prompt, options));
|
|
56
|
+
},
|
|
57
|
+
chat(messages, options = {}) {
|
|
58
|
+
assertSupportedGenerationOptions(options);
|
|
59
|
+
return pipeline.generate(messages, options);
|
|
60
|
+
},
|
|
61
|
+
async chatText(messages, options = {}) {
|
|
62
|
+
assertSupportedGenerationOptions(options);
|
|
63
|
+
const content = await collectText(pipeline.generate(messages, options));
|
|
64
|
+
const promptText = resolveChatPromptForUsage(pipeline, messages);
|
|
65
|
+
const promptTokens = countTokens(pipeline, promptText);
|
|
66
|
+
const completionTokens = countTokens(pipeline, content);
|
|
67
|
+
return {
|
|
68
|
+
content,
|
|
69
|
+
usage: {
|
|
70
|
+
promptTokens,
|
|
71
|
+
completionTokens,
|
|
72
|
+
totalTokens: promptTokens + completionTokens,
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
},
|
|
76
|
+
async loadLoRA(adapter) {
|
|
77
|
+
return loadLoRAAdapterForPipeline(pipeline, adapter);
|
|
78
|
+
},
|
|
79
|
+
async activateLoRAFromTrainingOutput(trainingOutput) {
|
|
80
|
+
return activateLoRAFromTrainingOutputForPipeline(pipeline, trainingOutput);
|
|
81
|
+
},
|
|
82
|
+
async unloadLoRA() {
|
|
83
|
+
return unloadLoRAAdapterForPipeline(pipeline);
|
|
84
|
+
},
|
|
85
|
+
async unload() {
|
|
86
|
+
await pipeline.unload();
|
|
87
|
+
},
|
|
88
|
+
get activeLoRA() {
|
|
89
|
+
return getActiveLoRAForPipeline(pipeline);
|
|
90
|
+
},
|
|
91
|
+
get loaded() {
|
|
92
|
+
return pipeline.isLoaded === true;
|
|
93
|
+
},
|
|
94
|
+
get modelId() {
|
|
95
|
+
return resolved.modelId;
|
|
96
|
+
},
|
|
97
|
+
get manifest() {
|
|
98
|
+
return pipeline.manifest;
|
|
99
|
+
},
|
|
100
|
+
get deviceInfo() {
|
|
101
|
+
return getKernelCapabilities()?.adapterInfo ?? null;
|
|
102
|
+
},
|
|
103
|
+
advanced: {
|
|
104
|
+
prefillKV(prompt, options = {}) {
|
|
105
|
+
assertSupportedGenerationOptions(options);
|
|
106
|
+
return pipeline.prefillKVOnly(prompt, options);
|
|
107
|
+
},
|
|
108
|
+
prefillWithLogits(prompt, options = {}) {
|
|
109
|
+
assertSupportedGenerationOptions(options);
|
|
110
|
+
return pipeline.prefillWithLogits(prompt, options);
|
|
111
|
+
},
|
|
112
|
+
decodeStepLogits(currentIds, options = {}) {
|
|
113
|
+
assertSupportedGenerationOptions(options);
|
|
114
|
+
return pipeline.decodeStepLogits(currentIds, options);
|
|
115
|
+
},
|
|
116
|
+
generateWithPrefixKV(prefix, prompt, options = {}) {
|
|
117
|
+
assertSupportedGenerationOptions(options);
|
|
118
|
+
return pipeline.generateWithPrefixKV(prefix, prompt, options);
|
|
119
|
+
},
|
|
120
|
+
},
|
|
121
|
+
};
|
|
122
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { RDRRManifest } from '../../formats/rdrr/index.js';
|
|
2
|
+
|
|
3
|
+
export interface DopplerLoadProgress {
|
|
4
|
+
phase: 'resolve' | 'manifest' | 'load' | 'ready';
|
|
5
|
+
percent: number;
|
|
6
|
+
message: string;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface DopplerModelSourceResolution {
|
|
10
|
+
modelId: string;
|
|
11
|
+
baseUrl: string | null;
|
|
12
|
+
manifest: RDRRManifest | null;
|
|
13
|
+
trace: Array<{ source: string; id: string; outcome: string }>;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export type DopplerModelSource =
|
|
17
|
+
| string
|
|
18
|
+
| {
|
|
19
|
+
url: string;
|
|
20
|
+
}
|
|
21
|
+
| {
|
|
22
|
+
manifest: RDRRManifest;
|
|
23
|
+
baseUrl?: string;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export interface DopplerLoadOptions {
|
|
27
|
+
onProgress?: (event: DopplerLoadProgress) => void;
|
|
28
|
+
runtimeConfig?: Record<string, unknown>;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export declare function createDefaultNodeLoadProgressLogger(): (event: DopplerLoadProgress) => void;
|
|
32
|
+
|
|
33
|
+
export declare function resolveLoadProgressHandlers(
|
|
34
|
+
options?: DopplerLoadOptions,
|
|
35
|
+
defaultLoadProgressLogger?: ((event: DopplerLoadProgress) => void) | null
|
|
36
|
+
): {
|
|
37
|
+
userProgress: ((event: DopplerLoadProgress) => void) | null;
|
|
38
|
+
pipelineProgress: ((event: DopplerLoadProgress) => void) | null;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
export declare function fetchManifestFromBaseUrl(baseUrl: string): Promise<RDRRManifest>;
|
|
42
|
+
|
|
43
|
+
export declare function resolveModelSource(model: DopplerModelSource): Promise<DopplerModelSourceResolution>;
|