doppler-gpu 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
package/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,26 @@ This changelog is package-facing and release-oriented. Entries before `0.1.7`
|
|
|
6
6
|
were retrofitted from package version history, release commits, and release
|
|
7
7
|
docs so the `0.1.x` line has one conventional npm-visible history surface.
|
|
8
8
|
|
|
9
|
+
## [0.2.1] - 2026-04-13
|
|
10
|
+
|
|
11
|
+
### Changed
|
|
12
|
+
|
|
13
|
+
- Bumped the package version past the already-published `0.2.0` release.
|
|
14
|
+
- Pointed the README demo CTA at the canonical public demo URL:
|
|
15
|
+
`https://d4da.com/doppler`.
|
|
16
|
+
- Clarified the README benchmark image scope so the hero chart stays tied to the
|
|
17
|
+
published MacBook Air M3 warm-cache lanes instead of implying whole-catalog
|
|
18
|
+
coverage.
|
|
19
|
+
- Expanded the README verified-model note to include Gemma 4 E2B alongside the
|
|
20
|
+
other verified local-artifact models outside the quickstart registry.
|
|
21
|
+
- Refreshed release-facing metadata so the release matrix now surfaces the
|
|
22
|
+
Gemma 4 E2B compare lane and current package version.
|
|
23
|
+
|
|
24
|
+
### Fixed
|
|
25
|
+
|
|
26
|
+
- Synced the public quickstart registry revisions with the canonical catalog for
|
|
27
|
+
`gemma-3-1b-it-q4k-ehf16-af32` and `qwen-3-5-0-8b-q4k-ehaf16`.
|
|
28
|
+
|
|
9
29
|
## [0.1.8] - 2026-03-18
|
|
10
30
|
|
|
11
31
|
### Changed
|
|
@@ -18,7 +38,7 @@ docs so the `0.1.x` line has one conventional npm-visible history surface.
|
|
|
18
38
|
- Aligned catalog, HF registry, and quickstart registry to the canonical
|
|
19
39
|
external support registry as single source of truth for HF revisions.
|
|
20
40
|
- Renamed all `.mjs` tool scripts to `.js` to match `"type": "module"` convention.
|
|
21
|
-
- Switched WebGPU optional dependency
|
|
41
|
+
- Switched WebGPU optional dependency to `webgpu@^0.4.0`.
|
|
22
42
|
- Pruned unused `verify:*` npm scripts for models no longer in the active set.
|
|
23
43
|
- Updated release-claim policy with newly verified models (LFM2, Qwen 3.5,
|
|
24
44
|
TranslateGemma variants).
|
package/README.md
CHANGED
|
@@ -1,13 +1,46 @@
|
|
|
1
1
|
# doppler-gpu
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Browser-native inference on raw WebGPU. Pure JS + WGSL.
|
|
4
4
|
|
|
5
|
-
**[Try the live demo](https://d4da.com)** | **[npm](https://www.npmjs.com/package/doppler-gpu)** | **[docs](https://github.com/clocksmith/doppler/blob/main/docs/INDEX.md)**
|
|
5
|
+
**[Try the live demo](https://d4da.com/doppler)** | **[npm](https://www.npmjs.com/package/doppler-gpu)** | **[docs](https://github.com/clocksmith/doppler/blob/main/docs/INDEX.md)**
|
|
6
6
|
|
|
7
|
-
](https://github.com/clocksmith/doppler/blob/main/docs/benchmark-methodology.md)
|
|
8
|
+
|
|
9
|
+
This chart is scoped to two published warm compare lanes: Gemma 4 E2B and
|
|
10
|
+
Gemma 3 1B. It is no longer the old Gemma 3 + LFM snapshot. Broader model
|
|
11
|
+
status and the surrounding compare evidence live in the support and release
|
|
12
|
+
matrices.
|
|
13
|
+
See the [benchmark methodology](https://github.com/clocksmith/doppler/blob/main/docs/benchmark-methodology.md).
|
|
8
14
|
|
|
9
15
|
## Quick start
|
|
10
16
|
|
|
17
|
+
### Browser
|
|
18
|
+
|
|
19
|
+
Use the live demo link above — it runs entirely in the browser with no server required. Models load into the browser cache and work offline after first download.
|
|
20
|
+
|
|
21
|
+
### CLI
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
npx doppler-gpu
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Downloads the default quickstart model, runs a local prompt, and prints the answer.
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
npx doppler-gpu "Summarize WebGPU in one sentence"
|
|
31
|
+
npx doppler-gpu --model qwen3-0.8b --prompt "Write a haiku about GPUs"
|
|
32
|
+
npx doppler-gpu --list-models
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
### Root API
|
|
36
|
+
|
|
37
|
+
The `doppler` facade is the primary app-facing API.
|
|
38
|
+
The root package intentionally stays small: it exports `doppler` and `DOPPLER_VERSION`.
|
|
39
|
+
Advanced surfaces now live on explicit subpaths such as `doppler-gpu/loaders`,
|
|
40
|
+
`doppler-gpu/generation`, `doppler-gpu/tooling`, and `doppler-gpu/orchestration`.
|
|
41
|
+
Support tiers for those subpaths are tracked in the subsystem support matrix rather
|
|
42
|
+
than assumed from export shape alone.
|
|
43
|
+
|
|
11
44
|
```js
|
|
12
45
|
import { doppler } from 'doppler-gpu';
|
|
13
46
|
|
|
@@ -19,52 +52,88 @@ for await (const token of model.generate('Describe WebGPU briefly')) {
|
|
|
19
52
|
|
|
20
53
|
// One-shot
|
|
21
54
|
const text = await model.generateText('Explain WebGPU in one sentence');
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
### OpenAI-compatible server
|
|
22
58
|
|
|
23
|
-
|
|
24
|
-
|
|
59
|
+
For existing apps, SDKs, and eval stacks that speak the OpenAI protocol:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
npx doppler-serve --model gemma3-270m --port 8080
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Then point any OpenAI client at `http://localhost:8080/v1`:
|
|
66
|
+
|
|
67
|
+
```js
|
|
68
|
+
import OpenAI from 'openai';
|
|
69
|
+
const client = new OpenAI({ baseURL: 'http://localhost:8080/v1', apiKey: 'unused' });
|
|
70
|
+
const response = await client.chat.completions.create({
|
|
71
|
+
model: 'gemma3-270m',
|
|
72
|
+
messages: [{ role: 'user', content: 'Hello' }],
|
|
73
|
+
});
|
|
25
74
|
```
|
|
26
75
|
|
|
27
|
-
|
|
76
|
+
This is a compatibility bridge — the core engine runs identically in the browser or Node.
|
|
77
|
+
|
|
78
|
+
Registry IDs resolve to hosted RDRR artifacts from `Clocksmith/rdrr` by default. See the [Root API guide](https://github.com/clocksmith/doppler/blob/main/docs/api/root.md).
|
|
79
|
+
|
|
80
|
+
## Support contract
|
|
81
|
+
|
|
82
|
+
Doppler keeps model support and subsystem support separate:
|
|
83
|
+
|
|
84
|
+
- [model support matrix](https://github.com/clocksmith/doppler/blob/main/docs/model-support-matrix.md): which models are verified right now
|
|
85
|
+
- [subsystem support matrix](https://github.com/clocksmith/doppler/blob/main/docs/subsystem-support-matrix.md): which runtime and API surfaces are `tier1`, `experimental`, or `internal-only`
|
|
86
|
+
|
|
87
|
+
The tier1 proof surface is the hosted browser demo, the root `doppler` API, the quickstart CLI, the OpenAI-compatible localhost server, and the verified text-inference path behind them.
|
|
28
88
|
|
|
29
89
|
## Why Doppler
|
|
30
90
|
|
|
31
|
-
**
|
|
91
|
+
**Browser-native.** Runs in a WebGPU browser tab with OPFS caching, so models stay available offline after the first load.
|
|
92
|
+
|
|
93
|
+
**JavaScript-first execution.** JSON resolves policy, JavaScript handles orchestration, and WGSL kernels handle compute. Kernel paths, dtype choices, and runtime behavior stay visible in the shipped source.
|
|
94
|
+
|
|
95
|
+
**Fast iteration.** JS, WGSL, and JSON changes run directly through the same stack used by the browser and Node surfaces, which keeps debugging and profiling close to real runtime behavior.
|
|
32
96
|
|
|
33
97
|
**`for await` streaming.** Generation uses a native `AsyncGenerator` that fits normal app control flow.
|
|
34
98
|
|
|
35
|
-
**LoRA hot-swap.**
|
|
99
|
+
**LoRA hot-swap.** Experimental advanced surface for swapping adapters at runtime without reloading the base model.
|
|
36
100
|
|
|
37
101
|
**Independent model instances.** Run multiple models concurrently. Each owns its pipeline, buffers, and KV cache.
|
|
38
102
|
|
|
39
|
-
##
|
|
103
|
+
## Quickstart-supported models
|
|
40
104
|
|
|
41
105
|
All models below are verified with deterministic greedy decoding on WebGPU hardware.
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
|
46
|
-
|
|
|
47
|
-
| Gemma 3
|
|
48
|
-
|
|
|
49
|
-
| EmbeddingGemma 300M | `
|
|
50
|
-
| Qwen 3.5 0.8B | `
|
|
51
|
-
| Qwen 3.5 2B | `
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
106
|
+
These registry IDs resolve to hosted RDRR artifacts automatically from the browser demo,
|
|
107
|
+
`npx doppler-gpu`, or `doppler.load(...)`.
|
|
108
|
+
|
|
109
|
+
| Model | Registry ID | Quant | Size | Family |
|
|
110
|
+
| --- | --- | --- | --- | --- |
|
|
111
|
+
| Gemma 3 270M IT | `gemma3-270m` | Q4K | 270M | Gemma |
|
|
112
|
+
| Gemma 3 1B IT | `gemma3-1b` | Q4K | 1B | Gemma |
|
|
113
|
+
| EmbeddingGemma 300M | `embeddinggemma-300m` | Q4K | 300M | Gemma |
|
|
114
|
+
| Qwen 3.5 0.8B | `qwen3-0.8b` | Q4K | 0.8B | Qwen |
|
|
115
|
+
| Qwen 3.5 2B | `qwen3-2b` | Q4K | 2B | Qwen |
|
|
116
|
+
|
|
117
|
+
Additional verified local-artifact models (TranslateGemma 4B, LFM2.5 1.2B) are
|
|
118
|
+
available outside the quickstart registry. Conversion configs exist for Gemma 4
|
|
119
|
+
MoE, Janus, and Sana but are not yet in the quickstart registry.
|
|
120
|
+
See the
|
|
121
|
+
[model support matrix](https://github.com/clocksmith/doppler/blob/main/docs/model-support-matrix.md).
|
|
122
|
+
Subsystem support tiers for direct-source inputs, advanced subpaths, diffusion,
|
|
123
|
+
energy, and training live in the
|
|
124
|
+
[subsystem support matrix](https://github.com/clocksmith/doppler/blob/main/docs/subsystem-support-matrix.md).
|
|
58
125
|
|
|
59
126
|
## Under the hood
|
|
60
127
|
|
|
61
128
|
- Sharded weight loading via OPFS moves multi-GB weights into VRAM without blocking the main thread.
|
|
62
129
|
- Quantized inference (Q4K, F16) runs practical model sizes on consumer GPUs.
|
|
130
|
+
- TurboQuant KV-cache profiles are available for quantized decode-cache runs.
|
|
63
131
|
- Kernel hot-swap between prefill and decode paths with zero graph recompilation.
|
|
64
132
|
- Config-driven runtime with explicit profiles, kernel-path selection, and sampling.
|
|
65
133
|
|
|
66
134
|
## Documentation
|
|
67
135
|
|
|
136
|
+
- npm quickstart: run `npx doppler-gpu --help`
|
|
68
137
|
- Docs index (canonical navigation): [docs/INDEX.md](https://github.com/clocksmith/doppler/blob/main/docs/INDEX.md)
|
|
69
138
|
- First-run workflow: [docs/getting-started.md](https://github.com/clocksmith/doppler/blob/main/docs/getting-started.md)
|
|
70
139
|
- CLI reference: [docs/cli.md](https://github.com/clocksmith/doppler/blob/main/docs/cli.md)
|
|
@@ -75,10 +144,9 @@ for details.
|
|
|
75
144
|
## Environment requirements
|
|
76
145
|
|
|
77
146
|
- WebGPU is required.
|
|
78
|
-
-
|
|
79
|
-
|
|
80
|
-
-
|
|
81
|
-
- Safari support is evolving.
|
|
147
|
+
- **Browser**: Current Chromium browsers with WebGPU enabled, including Chrome and Edge.
|
|
148
|
+
WebGPU shipped in Chrome/Edge 113+. Firefox and Safari support varies.
|
|
149
|
+
- **Node**: Requires a WebGPU provider (`webgpu` npm package). Installed automatically as an optional dependency.
|
|
82
150
|
|
|
83
151
|
## License
|
|
84
152
|
|
package/package.json
CHANGED
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "doppler-gpu",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "Browser-native WebGPU inference engine for local intent and inference loops",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"types": "src/index.d.ts",
|
|
7
7
|
"type": "module",
|
|
8
8
|
"bin": {
|
|
9
|
-
"doppler": "src/cli/doppler-cli.js"
|
|
9
|
+
"doppler": "src/cli/doppler-cli.js",
|
|
10
|
+
"doppler-gpu": "src/cli/doppler-quickstart.js",
|
|
11
|
+
"doppler-serve": "src/cli/doppler-serve.js"
|
|
10
12
|
},
|
|
11
13
|
"scripts": {
|
|
14
|
+
"quickstart": "node src/cli/doppler-quickstart.js",
|
|
15
|
+
"serve": "node src/cli/doppler-serve.js",
|
|
12
16
|
"cli": "node src/cli/doppler-cli.js",
|
|
13
17
|
"convert": "node src/cli/doppler-cli.js convert",
|
|
14
18
|
"manifest-refresh": "node tools/refresh-converted-manifest.js",
|
|
@@ -29,18 +33,20 @@
|
|
|
29
33
|
"bench:chart": "node ./benchmarks/vendors/compare-chart.js",
|
|
30
34
|
"bench:chart:readme": "node ./benchmarks/vendors/compare-chart.js --scenario readme-evidence",
|
|
31
35
|
"bench:architecture:chart": "node ./benchmarks/vendors/generate-architecture-overview-svg.js",
|
|
36
|
+
"bench:tjs:stage": "node tools/stage-tjs-model.js",
|
|
32
37
|
"ci:diffusion:contract": "node tools/ci-diffusion-contract-gates.js",
|
|
33
38
|
"ci:diffusion:contract:list": "node tools/ci-diffusion-contract-gates.js --list",
|
|
34
39
|
"ci:training:contract": "node tools/ci-training-contract-gates.js",
|
|
35
40
|
"ci:training:contract:list": "node tools/ci-training-contract-gates.js --list",
|
|
36
41
|
"training:contract:delta": "node tools/emit-training-contract-delta.js",
|
|
37
|
-
"training:workloads:verify": "node tools/verify-training-workload-packs.js --registry src/training/workload-packs/registry.json",
|
|
38
|
-
"training:report-ids:publish": "node tools/publish-training-report-ids.js --registry src/training/workload-packs/registry.json",
|
|
42
|
+
"training:workloads:verify": "node tools/verify-training-workload-packs.js --registry src/experimental/training/workload-packs/registry.json",
|
|
43
|
+
"training:report-ids:publish": "node tools/publish-training-report-ids.js --registry src/experimental/training/workload-packs/registry.json",
|
|
39
44
|
"distill:studio:mvp": "node tools/distill-studio-mvp.js",
|
|
40
45
|
"distill:quality-gate": "node tools/distill-studio-quality-gate.js",
|
|
41
46
|
"p2p:observability": "node tools/p2p-delivery-observability.js",
|
|
42
47
|
"p2p:drill": "node tools/p2p-resilience-drill.js",
|
|
43
48
|
"test": "npm run test:unit",
|
|
49
|
+
"typecheck:source-runtime": "tsc -p tsconfig.source-runtime.json --noEmit",
|
|
44
50
|
"test:unit": "node tools/run-node-tests.js --suite unit",
|
|
45
51
|
"test:gpu": "node tools/run-node-tests.js --suite gpu",
|
|
46
52
|
"test:coverage": "node tools/run-node-coverage.js",
|
|
@@ -53,6 +59,7 @@
|
|
|
53
59
|
"imports:check:browser": "node tools/check-browser-import-graph.js",
|
|
54
60
|
"api:docs:sync": "node tools/sync-api-docs.js",
|
|
55
61
|
"api:docs:check": "node tools/sync-api-docs.js --check",
|
|
62
|
+
"public:boundaries:check": "node tools/check-public-boundaries.js",
|
|
56
63
|
"verify:model": "node src/cli/doppler-cli.js verify",
|
|
57
64
|
"quickstart:sync": "node tools/sync-quickstart-registry.js",
|
|
58
65
|
"quickstart:check": "node tools/sync-quickstart-registry.js --check",
|
|
@@ -76,10 +83,10 @@
|
|
|
76
83
|
"registry:publish:hf": "node tools/publish-hf-registry-model.js",
|
|
77
84
|
"support:matrix:sync": "node tools/sync-model-support-matrix.js",
|
|
78
85
|
"support:matrix:check": "node tools/sync-model-support-matrix.js --check",
|
|
79
|
-
"
|
|
86
|
+
"support:subsystems:sync": "node tools/sync-subsystem-support-matrix.js",
|
|
87
|
+
"support:subsystems:check": "node tools/sync-subsystem-support-matrix.js --check",
|
|
88
|
+
"ci:catalog:check": "npm run registry:sync:scripts:check && npm run support:matrix:check && npm run support:subsystems:check && npm run registry:hf:check",
|
|
80
89
|
"verify:embeddinggemma-300m": "node tools/run-registry-verify.js embeddinggemma-300m",
|
|
81
|
-
"verify:gemma-3-1b-it-f16": "node tools/run-registry-verify.js gemma-3-1b-it-f16",
|
|
82
|
-
"verify:gemma-3-1b-it-f16-af32": "node tools/run-registry-verify.js gemma-3-1b-it-f16-af32",
|
|
83
90
|
"verify:gemma-3-1b-it-q4k-ehf16-af32": "node tools/run-registry-verify.js gemma-3-1b-it-q4k-ehf16-af32",
|
|
84
91
|
"verify:gemma-3-1b-it-wq4k-ef16-hf16": "node tools/run-registry-verify.js gemma-3-1b-it-wq4k-ef16-hf16",
|
|
85
92
|
"verify:gemma-3-270m-it-q4k-ehf16-af32": "node tools/run-registry-verify.js gemma-3-270m-it-q4k-ehf16-af32",
|
|
@@ -87,7 +94,6 @@
|
|
|
87
94
|
"verify:gemma-3-270m-it-wq4k-ef16-hf16": "node tools/run-registry-verify.js gemma-3-270m-it-wq4k-ef16-hf16",
|
|
88
95
|
"verify:gemma-3-270m-it-wq4k-ef16-hf16-f32": "node tools/run-registry-verify.js gemma-3-270m-it-wq4k-ef16-hf16-f32",
|
|
89
96
|
"verify:gemma3-1b": "node tools/run-registry-verify.js gemma3-1b",
|
|
90
|
-
"verify:gemma3-1b-f16": "node tools/run-registry-verify.js gemma3-1b-f16",
|
|
91
97
|
"verify:gemma3-270m": "node tools/run-registry-verify.js gemma3-270m",
|
|
92
98
|
"verify:google-embeddinggemma-300m": "node tools/run-registry-verify.js google-embeddinggemma-300m",
|
|
93
99
|
"verify:google-embeddinggemma-300m-q4k-ehf16-af32": "node tools/run-registry-verify.js google-embeddinggemma-300m-q4k-ehf16-af32",
|
|
@@ -110,10 +116,7 @@
|
|
|
110
116
|
"verify:qwen3-0.8b": "node tools/run-registry-verify.js qwen3-0.8b",
|
|
111
117
|
"verify:qwen3-2b": "node tools/run-registry-verify.js qwen3-2b",
|
|
112
118
|
"verify:translategemma": "node tools/run-registry-verify.js translategemma",
|
|
113
|
-
"verify:translategemma-1b-enes": "node tools/run-registry-verify.js translategemma-1b-enes",
|
|
114
119
|
"verify:translategemma-4b": "node tools/run-registry-verify.js translategemma-4b",
|
|
115
|
-
"verify:translategemma-4b-1b-enes": "node tools/run-registry-verify.js translategemma-4b-1b-enes",
|
|
116
|
-
"verify:translategemma-4b-1b-enes-q4k-ehf16-af32": "node tools/run-registry-verify.js translategemma-4b-1b-enes-q4k-ehf16-af32",
|
|
117
120
|
"verify:translategemma-4b-it-q4k-ehf16-af32": "node tools/run-registry-verify.js translategemma-4b-it-q4k-ehf16-af32",
|
|
118
121
|
"verify:translategemma-4b-it-wq4k-ef16-hf16": "node tools/run-registry-verify.js translategemma-4b-it-wq4k-ef16-hf16"
|
|
119
122
|
},
|
|
@@ -122,30 +125,35 @@
|
|
|
122
125
|
"types": "./src/index.d.ts",
|
|
123
126
|
"import": "./src/index.js"
|
|
124
127
|
},
|
|
125
|
-
"./provider": {
|
|
126
|
-
"types": "./src/client/doppler-provider.d.ts",
|
|
127
|
-
"import": "./src/client/doppler-provider.js"
|
|
128
|
-
},
|
|
129
128
|
"./tooling": {
|
|
130
129
|
"types": "./src/tooling-exports.d.ts",
|
|
131
130
|
"browser": "./src/tooling-exports.browser.js",
|
|
132
131
|
"import": "./src/tooling-exports.js"
|
|
133
132
|
},
|
|
134
|
-
"./
|
|
135
|
-
"types": "./src/
|
|
136
|
-
"
|
|
133
|
+
"./tooling-experimental": {
|
|
134
|
+
"types": "./src/tooling-experimental-exports.d.ts",
|
|
135
|
+
"browser": "./src/tooling-experimental-exports.browser.js",
|
|
136
|
+
"import": "./src/tooling-experimental-exports.js"
|
|
137
|
+
},
|
|
138
|
+
"./loaders": {
|
|
139
|
+
"types": "./src/loaders/index.d.ts",
|
|
140
|
+
"import": "./src/loaders/index.js"
|
|
141
|
+
},
|
|
142
|
+
"./orchestration": {
|
|
143
|
+
"types": "./src/experimental/orchestration/index.d.ts",
|
|
144
|
+
"import": "./src/experimental/orchestration/index.js"
|
|
137
145
|
},
|
|
138
146
|
"./generation": {
|
|
139
147
|
"types": "./src/generation/index.d.ts",
|
|
140
148
|
"import": "./src/generation/index.js"
|
|
141
149
|
},
|
|
142
150
|
"./diffusion": {
|
|
143
|
-
"types": "./src/diffusion/index.d.ts",
|
|
144
|
-
"import": "./src/diffusion/index.js"
|
|
151
|
+
"types": "./src/experimental/diffusion/index.d.ts",
|
|
152
|
+
"import": "./src/experimental/diffusion/index.js"
|
|
145
153
|
},
|
|
146
154
|
"./energy": {
|
|
147
|
-
"types": "./src/energy/index.d.ts",
|
|
148
|
-
"import": "./src/energy/index.js"
|
|
155
|
+
"types": "./src/experimental/energy/index.d.ts",
|
|
156
|
+
"import": "./src/experimental/energy/index.js"
|
|
149
157
|
}
|
|
150
158
|
},
|
|
151
159
|
"repository": {
|
|
@@ -180,16 +188,21 @@
|
|
|
180
188
|
"BRANDING.md",
|
|
181
189
|
"SECURITY.md",
|
|
182
190
|
"src/cli/doppler-cli.js",
|
|
183
|
-
"tools/convert-safetensors-node.js"
|
|
191
|
+
"tools/convert-safetensors-node.js",
|
|
192
|
+
"src/cli/config/doppler-quickstart.json",
|
|
193
|
+
"src/cli/doppler-quickstart.js",
|
|
194
|
+
"src/cli/doppler-serve.js"
|
|
184
195
|
],
|
|
185
196
|
"devDependencies": {
|
|
186
|
-
"@huggingface/transformers": "4.0.
|
|
197
|
+
"@huggingface/transformers": "^4.0.1",
|
|
198
|
+
"@types/node": "^25.6.0",
|
|
199
|
+
"@webgpu/types": "^0.1.69",
|
|
187
200
|
"jest": "^30.2.0",
|
|
188
201
|
"onnxruntime-web": "^1.24.1",
|
|
189
|
-
"playwright": "^1.58.2"
|
|
202
|
+
"playwright": "^1.58.2",
|
|
203
|
+
"typescript": "^6.0.2"
|
|
190
204
|
},
|
|
191
205
|
"optionalDependencies": {
|
|
192
|
-
"
|
|
193
|
-
"webgpu": "^0.3.8"
|
|
206
|
+
"webgpu": "^0.4.0"
|
|
194
207
|
}
|
|
195
208
|
}
|
package/src/cli/doppler-cli.js
CHANGED
|
@@ -561,36 +561,46 @@ export async function resolveNodeModelUrl(request, options = {}) {
|
|
|
561
561
|
}
|
|
562
562
|
|
|
563
563
|
const modelId = String(request.modelId);
|
|
564
|
+
const staticRootDir = resolveStaticRootDir(options);
|
|
564
565
|
const rdrrRoot = resolveRdrrRoot(options);
|
|
565
566
|
const externalModel = await resolveExternalModelDirectory(rdrrRoot, modelId);
|
|
567
|
+
const localCandidates = [
|
|
568
|
+
{
|
|
569
|
+
modelDir: path.join(staticRootDir, 'models', 'local', modelId),
|
|
570
|
+
manifestPath: path.join(staticRootDir, 'models', 'local', modelId, 'manifest.json'),
|
|
571
|
+
},
|
|
572
|
+
{
|
|
573
|
+
modelDir: path.join(staticRootDir, 'models', modelId),
|
|
574
|
+
manifestPath: path.join(staticRootDir, 'models', modelId, 'manifest.json'),
|
|
575
|
+
},
|
|
576
|
+
];
|
|
577
|
+
const candidates = [
|
|
578
|
+
...localCandidates,
|
|
579
|
+
{
|
|
580
|
+
modelDir: externalModel?.modelDir || path.join(rdrrRoot, modelId),
|
|
581
|
+
manifestPath: externalModel?.manifestPath || path.join(rdrrRoot, modelId, 'manifest.json'),
|
|
582
|
+
},
|
|
583
|
+
];
|
|
584
|
+
const { candidate, discoveredManifestCandidates } =
|
|
585
|
+
await findResolvableModelCandidate(candidates);
|
|
566
586
|
|
|
567
|
-
if (
|
|
568
|
-
const modelDir = externalModel.modelDir;
|
|
569
|
-
try {
|
|
570
|
-
const files = await fs.readdir(modelDir, { withFileTypes: true });
|
|
571
|
-
const hasShards = files.some((entry) =>
|
|
572
|
-
entry.isFile() && /^shard_\d+\.bin$/u.test(entry.name)
|
|
573
|
-
);
|
|
574
|
-
if (!hasShards) {
|
|
575
|
-
throw new Error(
|
|
576
|
-
`Model "${modelId}" found at ${modelDir} but no shard files (shard_*.bin) are present. `
|
|
577
|
-
+ 'Add shard files beside the manifest, or set request.modelUrl to a complete model directory.'
|
|
578
|
-
);
|
|
579
|
-
}
|
|
580
|
-
} catch (err) {
|
|
581
|
-
if (err.code === 'ENOENT' || err.code === 'EACCES') {
|
|
582
|
-
throw new Error(
|
|
583
|
-
`Model "${modelId}" resolved to ${modelDir} but the directory is not accessible: ${err.message}`
|
|
584
|
-
);
|
|
585
|
-
}
|
|
586
|
-
throw err;
|
|
587
|
-
}
|
|
587
|
+
if (candidate) {
|
|
588
588
|
return {
|
|
589
589
|
...request,
|
|
590
|
-
modelUrl: pathToFileURL(modelDir).href.replace(/\/$/, ''),
|
|
590
|
+
modelUrl: pathToFileURL(candidate.modelDir).href.replace(/\/$/, ''),
|
|
591
591
|
};
|
|
592
592
|
}
|
|
593
593
|
|
|
594
|
+
if (discoveredManifestCandidates.length > 0) {
|
|
595
|
+
const paths = discoveredManifestCandidates
|
|
596
|
+
.map((candidate) => candidate.modelDir)
|
|
597
|
+
.join(', ');
|
|
598
|
+
throw new Error(
|
|
599
|
+
`Model "${modelId}" was found, but no shard files (shard_*.bin) are present. ` +
|
|
600
|
+
`Checked: ${paths}. Add shard files beside the manifest, or set request.modelUrl to a complete model directory.`
|
|
601
|
+
);
|
|
602
|
+
}
|
|
603
|
+
|
|
594
604
|
const catalogEntry = await resolveCatalogEntry(modelId);
|
|
595
605
|
if (catalogEntry) {
|
|
596
606
|
const hfUrl = buildCatalogModelUrl(catalogEntry);
|
|
@@ -1321,6 +1331,12 @@ function printDeviceInfo(result) {
|
|
|
1321
1331
|
}
|
|
1322
1332
|
|
|
1323
1333
|
function printGpuPhases(metrics) {
|
|
1334
|
+
if (typeof metrics?.decodeMode === 'string' && metrics.decodeMode.length > 0) {
|
|
1335
|
+
const reason = typeof metrics?.batchGuardReason === 'string' && metrics.batchGuardReason.length > 0
|
|
1336
|
+
? ` guard=${metrics.batchGuardReason}`
|
|
1337
|
+
: '';
|
|
1338
|
+
console.log(`[decode] mode=${metrics.decodeMode}${reason}`);
|
|
1339
|
+
}
|
|
1324
1340
|
const gpu = metrics?.gpu;
|
|
1325
1341
|
if (!gpu) return;
|
|
1326
1342
|
const rm = gpu.decodeRecordMs?.median;
|
|
@@ -1334,6 +1350,34 @@ function printGpuPhases(metrics) {
|
|
|
1334
1350
|
if (Number.isFinite(pm) || Number.isFinite(dm)) {
|
|
1335
1351
|
console.log(`[gpu] prefill=${formatMs(pm)} decode=${formatMs(dm)} (median gpu time)`);
|
|
1336
1352
|
}
|
|
1353
|
+
const sts = gpu.singleTokenSubmitWaitMs?.median;
|
|
1354
|
+
const str = gpu.singleTokenReadbackWaitMs?.median;
|
|
1355
|
+
const sto = gpu.singleTokenOrchestrationMs?.median;
|
|
1356
|
+
if (Number.isFinite(sts) || Number.isFinite(str) || Number.isFinite(sto)) {
|
|
1357
|
+
console.log(
|
|
1358
|
+
`[gpu] single-token submit=${formatMs(sts)} readback=${formatMs(str)} ` +
|
|
1359
|
+
`orchestration=${formatMs(sto)} (median)`
|
|
1360
|
+
);
|
|
1361
|
+
}
|
|
1362
|
+
const batching = metrics?.batching;
|
|
1363
|
+
const batchedCalls = batching?.batchedForwardCalls?.median;
|
|
1364
|
+
const unbatchedCalls = batching?.unbatchedForwardCalls?.median;
|
|
1365
|
+
const batchedTime = batching?.totalBatchedTimeMs?.median;
|
|
1366
|
+
const unbatchedTime = batching?.totalUnbatchedTimeMs?.median;
|
|
1367
|
+
const submissions = batching?.gpuSubmissions?.median;
|
|
1368
|
+
if (
|
|
1369
|
+
Number.isFinite(batchedCalls)
|
|
1370
|
+
|| Number.isFinite(unbatchedCalls)
|
|
1371
|
+
|| Number.isFinite(batchedTime)
|
|
1372
|
+
|| Number.isFinite(unbatchedTime)
|
|
1373
|
+
|| Number.isFinite(submissions)
|
|
1374
|
+
) {
|
|
1375
|
+
console.log(
|
|
1376
|
+
`[batching] batched_calls=${formatNumber(batchedCalls)} unbatched_calls=${formatNumber(unbatchedCalls)} ` +
|
|
1377
|
+
`batched_time=${formatMs(batchedTime)} unbatched_time=${formatMs(unbatchedTime)} ` +
|
|
1378
|
+
`gpu_submissions=${formatNumber(submissions)}`
|
|
1379
|
+
);
|
|
1380
|
+
}
|
|
1337
1381
|
}
|
|
1338
1382
|
|
|
1339
1383
|
function printMemoryReport(result) {
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
export declare function parseQuickstartArgs(argv: string[]): {
|
|
2
|
+
model: string | null;
|
|
3
|
+
prompt: string | null;
|
|
4
|
+
maxTokens: string | null;
|
|
5
|
+
temperature: string | null;
|
|
6
|
+
json: boolean;
|
|
7
|
+
help: boolean;
|
|
8
|
+
listModels: boolean;
|
|
9
|
+
positionalPrompt: string | null;
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
export declare function readQuickstartConfig(): Promise<{
|
|
13
|
+
schemaVersion: number;
|
|
14
|
+
defaults: {
|
|
15
|
+
model: string;
|
|
16
|
+
prompt: string;
|
|
17
|
+
maxTokens: number;
|
|
18
|
+
temperature: number;
|
|
19
|
+
topK: number;
|
|
20
|
+
};
|
|
21
|
+
}>;
|
|
22
|
+
|
|
23
|
+
export declare function resolveQuickstartSettings(argv?: string[]): Promise<
|
|
24
|
+
| { action: 'help' }
|
|
25
|
+
| { action: 'list-models'; json: boolean }
|
|
26
|
+
| {
|
|
27
|
+
action: 'run';
|
|
28
|
+
json: boolean;
|
|
29
|
+
model: string;
|
|
30
|
+
prompt: string;
|
|
31
|
+
maxTokens: number;
|
|
32
|
+
temperature: number;
|
|
33
|
+
topK: number;
|
|
34
|
+
}
|
|
35
|
+
>;
|
|
36
|
+
|
|
37
|
+
export declare function runQuickstart(settings: {
|
|
38
|
+
json: boolean;
|
|
39
|
+
model: string;
|
|
40
|
+
prompt: string;
|
|
41
|
+
maxTokens: number;
|
|
42
|
+
temperature: number;
|
|
43
|
+
topK: number;
|
|
44
|
+
}): Promise<{
|
|
45
|
+
ok: true;
|
|
46
|
+
schemaVersion: number;
|
|
47
|
+
version: string;
|
|
48
|
+
requestedModel: string;
|
|
49
|
+
modelId: string;
|
|
50
|
+
prompt: string;
|
|
51
|
+
content: string;
|
|
52
|
+
usage: {
|
|
53
|
+
promptTokens: number;
|
|
54
|
+
completionTokens: number;
|
|
55
|
+
totalTokens: number;
|
|
56
|
+
};
|
|
57
|
+
elapsedMs: number;
|
|
58
|
+
deviceInfo: Record<string, unknown> | null;
|
|
59
|
+
}>;
|
|
60
|
+
|
|
61
|
+
export declare function main(argv?: string[]): Promise<void>;
|