doppler-gpu 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -24,6 +24,19 @@ export interface NodeBrowserCommandRunOptions {
|
|
|
24
24
|
onConsole?: (entry: { type: string; text: string }) => void;
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
+
export interface BrowserRelayLocalModelResolution {
|
|
28
|
+
relayRequest: ToolingCommandRequestInput;
|
|
29
|
+
staticMounts: Array<{
|
|
30
|
+
urlPrefix: string;
|
|
31
|
+
rootDir: string;
|
|
32
|
+
}>;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export declare function resolveLocalFileModelUrlForBrowserRelay(
|
|
36
|
+
commandRequest: ToolingCommandRequestInput,
|
|
37
|
+
options?: Pick<NodeBrowserCommandRunOptions, 'baseUrl' | 'staticMounts'>
|
|
38
|
+
): Promise<BrowserRelayLocalModelResolution>;
|
|
39
|
+
|
|
27
40
|
export declare function runBrowserCommandInNode(
|
|
28
41
|
commandRequest: ToolingCommandRequestInput,
|
|
29
42
|
options?: NodeBrowserCommandRunOptions
|
|
@@ -153,7 +153,7 @@ async function resolveFileForRequest(rootDir, mounts, requestPath) {
|
|
|
153
153
|
|
|
154
154
|
async function createStaticFileServer(options = {}) {
|
|
155
155
|
const rootDir = path.resolve(
|
|
156
|
-
options.rootDir || fileURLToPath(new URL('
|
|
156
|
+
options.rootDir || fileURLToPath(new URL('../..', import.meta.url))
|
|
157
157
|
);
|
|
158
158
|
const staticMounts = normalizeStaticMounts(options.staticMounts || []);
|
|
159
159
|
const host = String(options.host || DEFAULT_HOST);
|
|
@@ -313,6 +313,160 @@ function normalizeBaseUrl(value) {
|
|
|
313
313
|
}
|
|
314
314
|
}
|
|
315
315
|
|
|
316
|
+
function resolveLocalFileModelPath(modelUrl) {
|
|
317
|
+
return resolveLocalFileUrlPath(modelUrl, 'request.modelUrl');
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
function resolveLocalFileUrlPath(urlValue, fieldLabel) {
|
|
321
|
+
const normalized = asNonEmptyString(urlValue);
|
|
322
|
+
if (!normalized || !normalized.startsWith('file://')) {
|
|
323
|
+
return null;
|
|
324
|
+
}
|
|
325
|
+
try {
|
|
326
|
+
return fileURLToPath(normalized);
|
|
327
|
+
} catch (error) {
|
|
328
|
+
throw new Error(
|
|
329
|
+
`browser command: ${fieldLabel} must be a valid file:// URL when provided explicitly; ` +
|
|
330
|
+
`got ${JSON.stringify(urlValue)} (${error?.message || error}).`
|
|
331
|
+
);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
async function createLocalFileRelayMount(filePath, fieldLabel, urlPrefixRoot, options = {}) {
|
|
336
|
+
if (options.baseUrl) {
|
|
337
|
+
throw new Error(
|
|
338
|
+
`browser command: explicit local file:// ${fieldLabel} requires the relay-owned static server. ` +
|
|
339
|
+
'Remove run.browser.baseUrl or use a hosted URL instead.'
|
|
340
|
+
);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
let stats;
|
|
344
|
+
try {
|
|
345
|
+
stats = await fs.stat(filePath);
|
|
346
|
+
} catch (error) {
|
|
347
|
+
throw new Error(
|
|
348
|
+
`browser command: explicit local ${fieldLabel} "${options.originalUrl}" is not accessible: ${error?.message || error}.`
|
|
349
|
+
);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
const expectDirectory = options.kind === 'directory';
|
|
353
|
+
if (expectDirectory && !stats.isDirectory()) {
|
|
354
|
+
throw new Error(
|
|
355
|
+
`browser command: explicit local file:// ${fieldLabel} must point to a directory; got "${options.originalUrl}".`
|
|
356
|
+
);
|
|
357
|
+
}
|
|
358
|
+
if (!expectDirectory && !stats.isFile()) {
|
|
359
|
+
throw new Error(
|
|
360
|
+
`browser command: explicit local file:// ${fieldLabel} must point to a file; got "${options.originalUrl}".`
|
|
361
|
+
);
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
const fileName = path.basename(filePath) || options.fallbackName || 'asset';
|
|
365
|
+
const mountName = encodeURIComponent(fileName);
|
|
366
|
+
const mountPrefix = `${urlPrefixRoot}/${mountName}-${process.pid}-${Date.now()}`;
|
|
367
|
+
if (expectDirectory) {
|
|
368
|
+
return {
|
|
369
|
+
url: mountPrefix,
|
|
370
|
+
staticMount: {
|
|
371
|
+
urlPrefix: mountPrefix,
|
|
372
|
+
rootDir: filePath,
|
|
373
|
+
},
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
return {
|
|
378
|
+
url: `${mountPrefix}/${encodeURIComponent(fileName)}`,
|
|
379
|
+
staticMount: {
|
|
380
|
+
urlPrefix: mountPrefix,
|
|
381
|
+
rootDir: path.dirname(filePath),
|
|
382
|
+
},
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
export async function resolveLocalFileModelUrlForBrowserRelay(request, options = {}) {
|
|
387
|
+
const localModelPath = resolveLocalFileModelPath(request?.modelUrl);
|
|
388
|
+
const localRuntimeConfigPath = resolveLocalFileUrlPath(request?.runtimeConfigUrl, 'request.runtimeConfigUrl');
|
|
389
|
+
const localInferenceImagePath = resolveLocalFileUrlPath(
|
|
390
|
+
request?.inferenceInput?.image?.url,
|
|
391
|
+
'request.inferenceInput.image.url'
|
|
392
|
+
);
|
|
393
|
+
if (options.staticMounts != null && !Array.isArray(options.staticMounts)) {
|
|
394
|
+
throw new Error('browser command: staticMounts must be an array.');
|
|
395
|
+
}
|
|
396
|
+
const staticMounts = Array.isArray(options.staticMounts) ? [...options.staticMounts] : [];
|
|
397
|
+
if (!localModelPath && !localRuntimeConfigPath && !localInferenceImagePath) {
|
|
398
|
+
return {
|
|
399
|
+
relayRequest: request,
|
|
400
|
+
staticMounts,
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
let relayRequest = request;
|
|
404
|
+
const relayStaticMounts = [...staticMounts];
|
|
405
|
+
|
|
406
|
+
if (localModelPath) {
|
|
407
|
+
const modelRelayMount = await createLocalFileRelayMount(localModelPath, 'modelUrl', '/__doppler_local_model', {
|
|
408
|
+
...options,
|
|
409
|
+
originalUrl: request.modelUrl,
|
|
410
|
+
kind: 'directory',
|
|
411
|
+
fallbackName: 'model',
|
|
412
|
+
});
|
|
413
|
+
relayRequest = {
|
|
414
|
+
...relayRequest,
|
|
415
|
+
modelUrl: modelRelayMount.url,
|
|
416
|
+
};
|
|
417
|
+
relayStaticMounts.push(modelRelayMount.staticMount);
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
if (localRuntimeConfigPath) {
|
|
421
|
+
const runtimeConfigRelayMount = await createLocalFileRelayMount(
|
|
422
|
+
localRuntimeConfigPath,
|
|
423
|
+
'runtimeConfigUrl',
|
|
424
|
+
'/__doppler_local_runtime_config',
|
|
425
|
+
{
|
|
426
|
+
...options,
|
|
427
|
+
originalUrl: request.runtimeConfigUrl,
|
|
428
|
+
kind: 'file',
|
|
429
|
+
fallbackName: 'runtime-config.json',
|
|
430
|
+
}
|
|
431
|
+
);
|
|
432
|
+
relayRequest = {
|
|
433
|
+
...relayRequest,
|
|
434
|
+
runtimeConfigUrl: runtimeConfigRelayMount.url,
|
|
435
|
+
};
|
|
436
|
+
relayStaticMounts.push(runtimeConfigRelayMount.staticMount);
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
if (localInferenceImagePath) {
|
|
440
|
+
const imageRelayMount = await createLocalFileRelayMount(
|
|
441
|
+
localInferenceImagePath,
|
|
442
|
+
'inferenceInput.image.url',
|
|
443
|
+
'/__doppler_local_input_image',
|
|
444
|
+
{
|
|
445
|
+
...options,
|
|
446
|
+
originalUrl: request.inferenceInput.image.url,
|
|
447
|
+
kind: 'file',
|
|
448
|
+
fallbackName: 'input-image',
|
|
449
|
+
}
|
|
450
|
+
);
|
|
451
|
+
relayRequest = {
|
|
452
|
+
...relayRequest,
|
|
453
|
+
inferenceInput: {
|
|
454
|
+
...(relayRequest.inferenceInput || {}),
|
|
455
|
+
image: {
|
|
456
|
+
...(relayRequest.inferenceInput?.image || {}),
|
|
457
|
+
url: imageRelayMount.url,
|
|
458
|
+
},
|
|
459
|
+
},
|
|
460
|
+
};
|
|
461
|
+
relayStaticMounts.push(imageRelayMount.staticMount);
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
return {
|
|
465
|
+
relayRequest,
|
|
466
|
+
staticMounts: relayStaticMounts,
|
|
467
|
+
};
|
|
468
|
+
}
|
|
469
|
+
|
|
316
470
|
function normalizeBrowserArgs(value) {
|
|
317
471
|
if (value === undefined || value === null) return [];
|
|
318
472
|
if (!Array.isArray(value)) {
|
|
@@ -611,13 +765,19 @@ export async function runBrowserCommandInNode(commandRequest, options = {}) {
|
|
|
611
765
|
let useOpfsCache = options.opfsCache !== false;
|
|
612
766
|
let relayRequest = request;
|
|
613
767
|
const userDataDir = options.userDataDir || DEFAULT_OPFS_CACHE_DIR;
|
|
768
|
+
const baseUrl = normalizeBaseUrl(options.baseUrl);
|
|
769
|
+
const localModelRelayState = await resolveLocalFileModelUrlForBrowserRelay(relayRequest, {
|
|
770
|
+
baseUrl,
|
|
771
|
+
staticMounts: options.staticMounts,
|
|
772
|
+
});
|
|
773
|
+
relayRequest = localModelRelayState.relayRequest;
|
|
774
|
+
const relayStaticMounts = localModelRelayState.staticMounts;
|
|
614
775
|
|
|
615
776
|
if (options.wipeCacheBeforeLaunch && useOpfsCache) {
|
|
616
777
|
await fs.rm(userDataDir, { recursive: true, force: true }).catch(() => {});
|
|
617
778
|
}
|
|
618
779
|
|
|
619
780
|
const { chromium } = await import('playwright');
|
|
620
|
-
const baseUrl = normalizeBaseUrl(options.baseUrl);
|
|
621
781
|
// When OPFS caching is enabled, use a fixed port so the browser origin stays the same
|
|
622
782
|
// across runs (OPFS is origin-scoped). Without this, random ports create new origins.
|
|
623
783
|
const serverPort = options.port ?? (useOpfsCache ? DEFAULT_OPFS_CACHE_PORT : 0);
|
|
@@ -625,7 +785,7 @@ export async function runBrowserCommandInNode(commandRequest, options = {}) {
|
|
|
625
785
|
? null
|
|
626
786
|
: await createStaticFileServer({
|
|
627
787
|
rootDir: options.staticRootDir,
|
|
628
|
-
staticMounts:
|
|
788
|
+
staticMounts: relayStaticMounts,
|
|
629
789
|
host: options.host,
|
|
630
790
|
port: serverPort,
|
|
631
791
|
}).catch((error) => {
|
|
@@ -20,7 +20,6 @@ import {
|
|
|
20
20
|
getActiveKernelPathSource,
|
|
21
21
|
setActiveKernelPath,
|
|
22
22
|
} from '../config/kernel-path-loader.js';
|
|
23
|
-
import { runTrainingOperatorCommand } from '../training/operator-command.js';
|
|
24
23
|
|
|
25
24
|
function asOptionalPlainObject(value, label) {
|
|
26
25
|
if (value == null) return null;
|
|
@@ -56,6 +55,7 @@ function assertNoUnsupportedRuntimeInputs(request, reason = null) {
|
|
|
56
55
|
}
|
|
57
56
|
|
|
58
57
|
let runtimeModulesPromise = null;
|
|
58
|
+
let trainingOperatorModulesPromise = null;
|
|
59
59
|
|
|
60
60
|
function loadRuntimeModules() {
|
|
61
61
|
if (runtimeModulesPromise) {
|
|
@@ -71,6 +71,14 @@ function loadRuntimeModules() {
|
|
|
71
71
|
return runtimeModulesPromise;
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
+
async function loadTrainingOperatorModules() {
|
|
75
|
+
if (trainingOperatorModulesPromise) {
|
|
76
|
+
return trainingOperatorModulesPromise;
|
|
77
|
+
}
|
|
78
|
+
trainingOperatorModulesPromise = import('../experimental/training/operator-command.js');
|
|
79
|
+
return trainingOperatorModulesPromise;
|
|
80
|
+
}
|
|
81
|
+
|
|
74
82
|
export function hasNodeWebGPUSupport() {
|
|
75
83
|
const hasNavigatorGpu = typeof globalThis.navigator !== 'undefined' && !!globalThis.navigator.gpu;
|
|
76
84
|
const hasGpuEnums = typeof globalThis.GPUBufferUsage !== 'undefined' && typeof globalThis.GPUShaderStage !== 'undefined';
|
|
@@ -117,7 +125,7 @@ export async function runNodeCommand(commandRequest, options = {}) {
|
|
|
117
125
|
outputDir: request.outputDir,
|
|
118
126
|
converterConfig,
|
|
119
127
|
execution,
|
|
120
|
-
onProgress:
|
|
128
|
+
onProgress: validatedOptions.onProgress,
|
|
121
129
|
});
|
|
122
130
|
return createToolingSuccessEnvelope({
|
|
123
131
|
surface: 'node',
|
|
@@ -133,7 +141,8 @@ export async function runNodeCommand(commandRequest, options = {}) {
|
|
|
133
141
|
if (!gpuOptionalActions.has(request.action)) {
|
|
134
142
|
await assertNodeWebGPUSupport();
|
|
135
143
|
}
|
|
136
|
-
const
|
|
144
|
+
const training = await loadTrainingOperatorModules();
|
|
145
|
+
const result = await training.runTrainingOperatorCommand(request);
|
|
137
146
|
return createToolingSuccessEnvelope({
|
|
138
147
|
surface: 'node',
|
|
139
148
|
request,
|
|
@@ -142,7 +151,7 @@ export async function runNodeCommand(commandRequest, options = {}) {
|
|
|
142
151
|
}
|
|
143
152
|
|
|
144
153
|
if (request.command === 'diagnose') {
|
|
145
|
-
const result = await runDiagnoseCommand(request,
|
|
154
|
+
const result = await runDiagnoseCommand(request, validatedOptions);
|
|
146
155
|
return createToolingSuccessEnvelope({
|
|
147
156
|
surface: 'node',
|
|
148
157
|
request,
|
|
@@ -165,7 +174,7 @@ export async function runNodeCommand(commandRequest, options = {}) {
|
|
|
165
174
|
};
|
|
166
175
|
|
|
167
176
|
return runWithRuntimeIsolation(runtimeBridge, async () => {
|
|
168
|
-
await applyRuntimeInputs(request, runtimeBridge,
|
|
177
|
+
await applyRuntimeInputs(request, runtimeBridge, validatedOptions.runtimeLoadOptions || {});
|
|
169
178
|
const result = await modules.harness.runBrowserSuite(buildSuiteOptions(request, 'node'));
|
|
170
179
|
|
|
171
180
|
return createToolingSuccessEnvelope({
|
|
@@ -162,6 +162,88 @@ function normalizeWorkerTransformResult(result, tensor) {
|
|
|
162
162
|
};
|
|
163
163
|
}
|
|
164
164
|
|
|
165
|
+
const MAX_NODE_CONVERT_BUFFER_BYTES = 0xffff_ffff;
|
|
166
|
+
|
|
167
|
+
function resolveRowChunkTransformPlan(input) {
|
|
168
|
+
const tensor = input?.tensor;
|
|
169
|
+
const execution = input?.execution;
|
|
170
|
+
const transformContext = input?.transformContext ?? {};
|
|
171
|
+
const resolveTensorTargetQuant = input?.resolveTensorTargetQuant;
|
|
172
|
+
const normalizeStorageQuant = input?.normalizeStorageQuant;
|
|
173
|
+
const shouldQuantize = input?.shouldQuantize;
|
|
174
|
+
const tensorByteLength = Number(input?.tensorByteLength ?? 0);
|
|
175
|
+
|
|
176
|
+
if (!tensor || !execution) {
|
|
177
|
+
throw new Error('node convert: row chunk transform plan requires tensor and execution.');
|
|
178
|
+
}
|
|
179
|
+
if (typeof resolveTensorTargetQuant !== 'function' || typeof normalizeStorageQuant !== 'function') {
|
|
180
|
+
throw new Error('node convert: row chunk transform plan requires quantization helpers.');
|
|
181
|
+
}
|
|
182
|
+
if (typeof shouldQuantize !== 'function') {
|
|
183
|
+
throw new Error('node convert: row chunk transform plan requires shouldQuantize().');
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const sourceDtype = String(tensor.dtype || '').toUpperCase();
|
|
187
|
+
const sourceQuant = normalizeStorageQuant(sourceDtype);
|
|
188
|
+
const tensorTargetQuant = resolveTensorTargetQuant(
|
|
189
|
+
tensor.name,
|
|
190
|
+
transformContext.targetQuant,
|
|
191
|
+
transformContext.quantizationInfo ?? null
|
|
192
|
+
);
|
|
193
|
+
const is2D = Array.isArray(tensor.shape) && tensor.shape.length === 2;
|
|
194
|
+
const rows = is2D ? tensor.shape[0] : 0;
|
|
195
|
+
const cols = is2D ? tensor.shape[1] : 0;
|
|
196
|
+
const sourceBytesPerElement = getDtypeBytes(sourceDtype);
|
|
197
|
+
const q4kLayout = String(transformContext.q4kLayout || 'row').trim().toLowerCase() === 'col'
|
|
198
|
+
? 'col'
|
|
199
|
+
: 'row';
|
|
200
|
+
const canChunkRows = (
|
|
201
|
+
is2D
|
|
202
|
+
&& rows > 0
|
|
203
|
+
&& cols > 0
|
|
204
|
+
&& sourceBytesPerElement != null
|
|
205
|
+
&& sourceQuant !== 'q4k'
|
|
206
|
+
&& tensorByteLength >= execution.rowChunkMinTensorBytes
|
|
207
|
+
&& !(tensorTargetQuant === 'q4k' && q4kLayout === 'col')
|
|
208
|
+
);
|
|
209
|
+
const jobMode = selectRuleValue('converter', 'execution', 'jobMode', {
|
|
210
|
+
workers: execution.effectiveWorkers,
|
|
211
|
+
canChunkRows,
|
|
212
|
+
});
|
|
213
|
+
if (jobMode !== 'row_chunks' || !canChunkRows) {
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
const rowChunkRows = execution.rowChunkRows
|
|
218
|
+
?? selectRuleValue('converter', 'execution', 'rowChunkRows', {
|
|
219
|
+
workers: execution.effectiveWorkers,
|
|
220
|
+
canChunkRows,
|
|
221
|
+
});
|
|
222
|
+
if (!Number.isInteger(rowChunkRows) || rowChunkRows < 1) {
|
|
223
|
+
return null;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
const rowSourceBytes = cols * sourceBytesPerElement;
|
|
227
|
+
if (!Number.isInteger(rowSourceBytes) || rowSourceBytes < 1) {
|
|
228
|
+
return null;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
const forceQuantizeDecision = tensorTargetQuant === 'q4k'
|
|
232
|
+
? shouldQuantize(tensor.name, tensor.shape, {
|
|
233
|
+
quantizeEmbeddings: Boolean(transformContext.quantizeEmbeddings),
|
|
234
|
+
modulesToNotConvert: transformContext.modulesToNotConvert ?? null,
|
|
235
|
+
})
|
|
236
|
+
: null;
|
|
237
|
+
|
|
238
|
+
return {
|
|
239
|
+
rows,
|
|
240
|
+
cols,
|
|
241
|
+
rowChunkRows,
|
|
242
|
+
rowSourceBytes,
|
|
243
|
+
forceQuantizeDecision,
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
|
|
165
247
|
async function mapWithConcurrency(items, concurrency, mapper) {
|
|
166
248
|
if (!Array.isArray(items) || items.length === 0) return [];
|
|
167
249
|
const workerCount = Math.max(1, Math.min(concurrency, items.length));
|
|
@@ -679,76 +761,28 @@ function createNodeTensorTransformer(options) {
|
|
|
679
761
|
if (!tensor || !(tensorData instanceof Uint8Array)) {
|
|
680
762
|
throw new Error('node convert: invalid tensor transform input.');
|
|
681
763
|
}
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
const is2D = Array.isArray(tensor.shape) && tensor.shape.length === 2;
|
|
692
|
-
const rows = is2D ? tensor.shape[0] : 0;
|
|
693
|
-
const cols = is2D ? tensor.shape[1] : 0;
|
|
694
|
-
const sourceBytesPerElement = getDtypeBytes(sourceDtype);
|
|
695
|
-
const q4kLayout = String(transformContext.q4kLayout || 'row').trim().toLowerCase() === 'col'
|
|
696
|
-
? 'col'
|
|
697
|
-
: 'row';
|
|
698
|
-
const canChunkRows = (
|
|
699
|
-
is2D
|
|
700
|
-
&& rows > 0
|
|
701
|
-
&& cols > 0
|
|
702
|
-
&& sourceBytesPerElement != null
|
|
703
|
-
&& sourceQuant !== 'q4k'
|
|
704
|
-
&& tensorData.byteLength >= execution.rowChunkMinTensorBytes
|
|
705
|
-
&& !(tensorTargetQuant === 'q4k' && q4kLayout === 'col')
|
|
706
|
-
);
|
|
707
|
-
|
|
708
|
-
const jobMode = selectRuleValue('converter', 'execution', 'jobMode', {
|
|
709
|
-
workers: execution.effectiveWorkers,
|
|
710
|
-
canChunkRows,
|
|
764
|
+
const chunkPlan = resolveRowChunkTransformPlan({
|
|
765
|
+
tensor,
|
|
766
|
+
tensorByteLength: tensorData.byteLength,
|
|
767
|
+
execution,
|
|
768
|
+
transformContext,
|
|
769
|
+
resolveTensorTargetQuant,
|
|
770
|
+
normalizeStorageQuant,
|
|
771
|
+
shouldQuantize,
|
|
711
772
|
});
|
|
712
773
|
|
|
713
|
-
if (
|
|
714
|
-
const transformed = await pool.transformTensor(tensor, tensorData, transformContext);
|
|
715
|
-
const normalized = normalizeWorkerTransformResult(transformed, tensor);
|
|
716
|
-
reportProgress?.(tensorData.byteLength, tensorData.byteLength);
|
|
717
|
-
return normalized;
|
|
718
|
-
}
|
|
719
|
-
|
|
720
|
-
const rowChunkRows = execution.rowChunkRows
|
|
721
|
-
?? selectRuleValue('converter', 'execution', 'rowChunkRows', {
|
|
722
|
-
workers: execution.effectiveWorkers,
|
|
723
|
-
canChunkRows,
|
|
724
|
-
});
|
|
725
|
-
if (!Number.isInteger(rowChunkRows) || rowChunkRows < 1) {
|
|
726
|
-
const transformed = await pool.transformTensor(tensor, tensorData, transformContext);
|
|
727
|
-
const normalized = normalizeWorkerTransformResult(transformed, tensor);
|
|
728
|
-
reportProgress?.(tensorData.byteLength, tensorData.byteLength);
|
|
729
|
-
return normalized;
|
|
730
|
-
}
|
|
731
|
-
|
|
732
|
-
const rowSourceBytes = cols * sourceBytesPerElement;
|
|
733
|
-
if (!Number.isInteger(rowSourceBytes) || rowSourceBytes < 1) {
|
|
774
|
+
if (!chunkPlan) {
|
|
734
775
|
const transformed = await pool.transformTensor(tensor, tensorData, transformContext);
|
|
735
776
|
const normalized = normalizeWorkerTransformResult(transformed, tensor);
|
|
736
777
|
reportProgress?.(tensorData.byteLength, tensorData.byteLength);
|
|
737
778
|
return normalized;
|
|
738
779
|
}
|
|
739
780
|
|
|
740
|
-
const forceQuantizeDecision = tensorTargetQuant === 'q4k'
|
|
741
|
-
? shouldQuantize(tensor.name, tensor.shape, {
|
|
742
|
-
quantizeEmbeddings: Boolean(transformContext.quantizeEmbeddings),
|
|
743
|
-
modulesToNotConvert: transformContext.modulesToNotConvert ?? null,
|
|
744
|
-
})
|
|
745
|
-
: null;
|
|
746
|
-
|
|
747
781
|
const chunks = [];
|
|
748
|
-
for (let rowStart = 0; rowStart < rows; rowStart += rowChunkRows) {
|
|
749
|
-
const rowCount = Math.min(rowChunkRows, rows - rowStart);
|
|
750
|
-
const start = rowStart * rowSourceBytes;
|
|
751
|
-
const end = start + (rowCount * rowSourceBytes);
|
|
782
|
+
for (let rowStart = 0; rowStart < chunkPlan.rows; rowStart += chunkPlan.rowChunkRows) {
|
|
783
|
+
const rowCount = Math.min(chunkPlan.rowChunkRows, chunkPlan.rows - rowStart);
|
|
784
|
+
const start = rowStart * chunkPlan.rowSourceBytes;
|
|
785
|
+
const end = start + (rowCount * chunkPlan.rowSourceBytes);
|
|
752
786
|
chunks.push({ rowStart, rowCount, start, end });
|
|
753
787
|
}
|
|
754
788
|
|
|
@@ -765,11 +799,11 @@ function createNodeTensorTransformer(options) {
|
|
|
765
799
|
const chunkTensorData = tensorData.subarray(chunk.start, chunk.end);
|
|
766
800
|
const chunkTensor = {
|
|
767
801
|
...tensor,
|
|
768
|
-
shape: [chunk.rowCount, cols],
|
|
802
|
+
shape: [chunk.rowCount, chunkPlan.cols],
|
|
769
803
|
};
|
|
770
804
|
const transformed = await pool.transformTensor(chunkTensor, chunkTensorData, {
|
|
771
805
|
...transformContext,
|
|
772
|
-
forceQuantizeDecision,
|
|
806
|
+
forceQuantizeDecision: chunkPlan.forceQuantizeDecision,
|
|
773
807
|
});
|
|
774
808
|
const normalized = normalizeWorkerTransformResult(transformed, chunkTensor);
|
|
775
809
|
processedBytes += chunkTensorData.byteLength;
|
|
@@ -813,6 +847,120 @@ function createNodeTensorTransformer(options) {
|
|
|
813
847
|
};
|
|
814
848
|
}
|
|
815
849
|
|
|
850
|
+
function createNodeLargeTensorTransformer(options) {
|
|
851
|
+
const pool = options?.pool;
|
|
852
|
+
const execution = options?.execution;
|
|
853
|
+
const readRange = options?.readRange;
|
|
854
|
+
const resolveTensorTargetQuant = options?.resolveTensorTargetQuant;
|
|
855
|
+
const normalizeStorageQuant = options?.normalizeStorageQuant;
|
|
856
|
+
const shouldQuantize = options?.shouldQuantize;
|
|
857
|
+
|
|
858
|
+
if (!pool || typeof pool.transformTensor !== 'function' || !execution || typeof readRange !== 'function') {
|
|
859
|
+
throw new Error('node convert: invalid large tensor transformer setup.');
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
return async function largeTensorTransformer(input) {
|
|
863
|
+
const tensor = input?.tensor;
|
|
864
|
+
const transformContext = input?.transformContext ?? {};
|
|
865
|
+
const reportProgress = typeof input?.reportProgress === 'function'
|
|
866
|
+
? input.reportProgress
|
|
867
|
+
: null;
|
|
868
|
+
const writeChunk = typeof input?.writeChunk === 'function'
|
|
869
|
+
? input.writeChunk
|
|
870
|
+
: null;
|
|
871
|
+
|
|
872
|
+
if (!tensor || typeof tensor !== 'object') {
|
|
873
|
+
throw new Error('node convert: invalid large tensor transform input.');
|
|
874
|
+
}
|
|
875
|
+
if (!writeChunk) {
|
|
876
|
+
throw new Error('node convert: large tensor transform requires writeChunk().');
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
const tensorByteLength = Number(tensor?.size ?? 0);
|
|
880
|
+
const chunkPlan = resolveRowChunkTransformPlan({
|
|
881
|
+
tensor,
|
|
882
|
+
tensorByteLength,
|
|
883
|
+
execution,
|
|
884
|
+
transformContext,
|
|
885
|
+
resolveTensorTargetQuant,
|
|
886
|
+
normalizeStorageQuant,
|
|
887
|
+
shouldQuantize,
|
|
888
|
+
});
|
|
889
|
+
if (!chunkPlan) {
|
|
890
|
+
throw new Error(
|
|
891
|
+
`node convert: tensor "${tensor.name}" is ${tensorByteLength} bytes and exceeds the single-buffer limit, ` +
|
|
892
|
+
'but it is not eligible for row-chunked conversion.'
|
|
893
|
+
);
|
|
894
|
+
}
|
|
895
|
+
if (chunkPlan.rowSourceBytes > MAX_NODE_CONVERT_BUFFER_BYTES) {
|
|
896
|
+
throw new Error(
|
|
897
|
+
`node convert: tensor "${tensor.name}" cannot be row-chunked because each source row is ` +
|
|
898
|
+
`${chunkPlan.rowSourceBytes} bytes, above the single-buffer limit ${MAX_NODE_CONVERT_BUFFER_BYTES}.`
|
|
899
|
+
);
|
|
900
|
+
}
|
|
901
|
+
const maxRowsPerRead = Math.floor(MAX_NODE_CONVERT_BUFFER_BYTES / chunkPlan.rowSourceBytes);
|
|
902
|
+
if (maxRowsPerRead < 1) {
|
|
903
|
+
throw new Error(
|
|
904
|
+
`node convert: tensor "${tensor.name}" cannot be row-chunked under the current single-buffer limit.`
|
|
905
|
+
);
|
|
906
|
+
}
|
|
907
|
+
if (chunkPlan.rowChunkRows > maxRowsPerRead) {
|
|
908
|
+
throw new Error(
|
|
909
|
+
`node convert: execution.rowChunkRows=${chunkPlan.rowChunkRows} is too large for tensor "${tensor.name}". ` +
|
|
910
|
+
`Use ${maxRowsPerRead} rows or fewer for streamed conversion.`
|
|
911
|
+
);
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
let processedBytes = 0;
|
|
915
|
+
let outDtype = null;
|
|
916
|
+
let outLayout = null;
|
|
917
|
+
|
|
918
|
+
for (let rowStart = 0; rowStart < chunkPlan.rows; rowStart += chunkPlan.rowChunkRows) {
|
|
919
|
+
const rowCount = Math.min(chunkPlan.rowChunkRows, chunkPlan.rows - rowStart);
|
|
920
|
+
const chunkOffset = rowStart * chunkPlan.rowSourceBytes;
|
|
921
|
+
const chunkLength = rowCount * chunkPlan.rowSourceBytes;
|
|
922
|
+
const chunkTensor = {
|
|
923
|
+
...tensor,
|
|
924
|
+
shape: [rowCount, chunkPlan.cols],
|
|
925
|
+
size: chunkLength,
|
|
926
|
+
};
|
|
927
|
+
const rawChunk = await readRange(
|
|
928
|
+
tensor.sourcePath,
|
|
929
|
+
tensor.offset + chunkOffset,
|
|
930
|
+
chunkLength
|
|
931
|
+
);
|
|
932
|
+
const chunkTensorData = new Uint8Array(rawChunk);
|
|
933
|
+
const transformed = await pool.transformTensor(chunkTensor, chunkTensorData, {
|
|
934
|
+
...transformContext,
|
|
935
|
+
forceQuantizeDecision: chunkPlan.forceQuantizeDecision,
|
|
936
|
+
});
|
|
937
|
+
const normalized = normalizeWorkerTransformResult(transformed, chunkTensor);
|
|
938
|
+
if (outDtype == null) {
|
|
939
|
+
outDtype = normalized.outDtype ?? tensor.dtype;
|
|
940
|
+
outLayout = normalized.outLayout ?? null;
|
|
941
|
+
} else {
|
|
942
|
+
if ((normalized.outDtype ?? tensor.dtype) !== outDtype) {
|
|
943
|
+
throw new Error(`node convert: inconsistent streamed chunk dtype for ${tensor.name}.`);
|
|
944
|
+
}
|
|
945
|
+
if ((normalized.outLayout ?? null) !== outLayout) {
|
|
946
|
+
throw new Error(`node convert: inconsistent streamed chunk layout for ${tensor.name}.`);
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
await writeChunk(normalized);
|
|
950
|
+
processedBytes += chunkLength;
|
|
951
|
+
reportProgress?.(
|
|
952
|
+
Math.min(processedBytes, tensorByteLength),
|
|
953
|
+
tensorByteLength
|
|
954
|
+
);
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
return {
|
|
958
|
+
outDtype: outDtype ?? tensor.dtype,
|
|
959
|
+
outLayout: outLayout ?? null,
|
|
960
|
+
};
|
|
961
|
+
};
|
|
962
|
+
}
|
|
963
|
+
|
|
816
964
|
export async function convertSafetensorsDirectory(options) {
|
|
817
965
|
const inputDir = assertPath(options?.inputDir, 'inputDir');
|
|
818
966
|
const outputDirOverride = readOptionalNonEmptyString(options?.outputDir);
|
|
@@ -1200,6 +1348,7 @@ export async function convertSafetensorsDirectory(options) {
|
|
|
1200
1348
|
let workerTensorTransformer = null;
|
|
1201
1349
|
let gpuTensorTransformer = null;
|
|
1202
1350
|
let tensorTransformer = null;
|
|
1351
|
+
let largeTensorTransformer = null;
|
|
1203
1352
|
let result = null;
|
|
1204
1353
|
try {
|
|
1205
1354
|
if (executionPlan.useGpuCast) {
|
|
@@ -1230,6 +1379,19 @@ export async function convertSafetensorsDirectory(options) {
|
|
|
1230
1379
|
shouldQuantize,
|
|
1231
1380
|
});
|
|
1232
1381
|
}
|
|
1382
|
+
const chunkTransformPool = workerPool ?? {
|
|
1383
|
+
async transformTensor(tensor, tensorData, transformContext) {
|
|
1384
|
+
return transformTensorBytes(tensor, tensorData, transformContext);
|
|
1385
|
+
},
|
|
1386
|
+
};
|
|
1387
|
+
largeTensorTransformer = createNodeLargeTensorTransformer({
|
|
1388
|
+
pool: chunkTransformPool,
|
|
1389
|
+
execution: executionPlan,
|
|
1390
|
+
readRange: fileRangeReader.readRange,
|
|
1391
|
+
resolveTensorTargetQuant,
|
|
1392
|
+
normalizeStorageQuant,
|
|
1393
|
+
shouldQuantize,
|
|
1394
|
+
});
|
|
1233
1395
|
if (gpuTensorTransformer || workerTensorTransformer) {
|
|
1234
1396
|
tensorTransformer = async (input) => {
|
|
1235
1397
|
if (gpuTensorTransformer) {
|
|
@@ -1268,6 +1430,7 @@ export async function convertSafetensorsDirectory(options) {
|
|
|
1268
1430
|
inference,
|
|
1269
1431
|
converterConfig,
|
|
1270
1432
|
tensorTransformer,
|
|
1433
|
+
largeTensorTransformer,
|
|
1271
1434
|
onProgress(update) {
|
|
1272
1435
|
onProgress?.(toNodeProgress(update));
|
|
1273
1436
|
},
|
|
@@ -1,17 +1,22 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { RuntimeConfigSchema } from '../config/schema/index.js';
|
|
2
2
|
import type { PipelineStorageContext } from '../inference/pipelines/text/init.js';
|
|
3
|
+
import type { RuntimeModelContract } from '../inference/runtime-model.js';
|
|
4
|
+
import type { DirectSourceRuntimeKind } from './source-artifact-adapter.js';
|
|
3
5
|
|
|
4
6
|
export interface ResolveNodeSourceRuntimeBundleOptions {
|
|
5
7
|
inputPath: string;
|
|
6
8
|
modelId?: string | null;
|
|
7
9
|
verifyHashes?: boolean;
|
|
10
|
+
runtimeConfig?: RuntimeConfigSchema | null;
|
|
8
11
|
}
|
|
9
12
|
|
|
10
13
|
export interface NodeSourceRuntimeBundle {
|
|
11
|
-
|
|
14
|
+
model: RuntimeModelContract;
|
|
15
|
+
manifest: RuntimeModelContract;
|
|
12
16
|
storageContext: PipelineStorageContext;
|
|
13
|
-
sourceKind:
|
|
17
|
+
sourceKind: DirectSourceRuntimeKind;
|
|
14
18
|
sourceRoot: string;
|
|
19
|
+
resolvedMemoryBudgetBytes: number | null;
|
|
15
20
|
}
|
|
16
21
|
|
|
17
22
|
export declare function resolveNodeSourceRuntimeBundle(
|