doppler-gpu 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -1,47 +1,34 @@
|
|
|
1
1
|
import { createReadStream } from 'node:fs';
|
|
2
2
|
import fs from 'node:fs/promises';
|
|
3
3
|
import { createHash } from 'node:crypto';
|
|
4
|
+
import { totalmem } from 'node:os';
|
|
4
5
|
import path from 'node:path';
|
|
5
6
|
import {
|
|
6
7
|
HEADER_READ_SIZE,
|
|
7
|
-
createConverterConfig,
|
|
8
|
-
DEFAULT_EXECUTION_V1_SESSION_DEFAULTS,
|
|
9
8
|
} from '../config/schema/index.js';
|
|
9
|
+
import { getRuntimeConfig } from '../config/runtime.js';
|
|
10
10
|
import { extractArchitecture } from '../converter/core.js';
|
|
11
|
-
import {
|
|
12
|
-
inferSourceWeightQuantization,
|
|
13
|
-
resolveConversionPlan,
|
|
14
|
-
resolveConvertedModelId,
|
|
15
|
-
} from '../converter/conversion-plan.js';
|
|
16
11
|
import { parseGGUFModel } from '../converter/parsers/gguf.js';
|
|
17
12
|
import { parseTransformerModel } from '../converter/parsers/transformer.js';
|
|
18
13
|
import { parseGGUFHeader } from '../formats/gguf/types.js';
|
|
19
14
|
import { parseSafetensorsHeader } from '../formats/safetensors/types.js';
|
|
15
|
+
import { parseTFLiteFromSource } from '../formats/tflite/types.js';
|
|
20
16
|
import { log } from '../debug/index.js';
|
|
17
|
+
import { formatBytes } from '../storage/quota.js';
|
|
21
18
|
import {
|
|
22
|
-
buildSourceRuntimeBundle,
|
|
23
19
|
createSourceStorageContext,
|
|
24
20
|
} from './source-runtime-bundle.js';
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
const
|
|
36
|
-
steps: [],
|
|
37
|
-
};
|
|
38
|
-
|
|
39
|
-
function cloneSessionDefaults() {
|
|
40
|
-
if (typeof structuredClone === 'function') {
|
|
41
|
-
return structuredClone(DEFAULT_EXECUTION_V1_SESSION_DEFAULTS);
|
|
42
|
-
}
|
|
43
|
-
return JSON.parse(JSON.stringify(DEFAULT_EXECUTION_V1_SESSION_DEFAULTS));
|
|
44
|
-
}
|
|
21
|
+
import {
|
|
22
|
+
inferSourceQuantizationForSourceRuntime,
|
|
23
|
+
resolveSourceRuntimeBundleFromParsedArtifact,
|
|
24
|
+
} from './source-artifact-adapter.js';
|
|
25
|
+
import {
|
|
26
|
+
LITERT_PACKAGE_SOURCE_KIND_LITERTLM,
|
|
27
|
+
LITERT_PACKAGE_SOURCE_KIND_TASK,
|
|
28
|
+
appendLiteRTPackageVirtualFiles,
|
|
29
|
+
resolveLiteRTPackageParsedArtifact,
|
|
30
|
+
} from './litert-package-runtime.js';
|
|
31
|
+
const MAX_NODE_READ_BYTES = 64 * 1024 * 1024;
|
|
45
32
|
|
|
46
33
|
function toArrayBuffer(value, label) {
|
|
47
34
|
if (value instanceof ArrayBuffer) {
|
|
@@ -61,23 +48,16 @@ function isGgufPath(filePath) {
|
|
|
61
48
|
return String(filePath || '').toLowerCase().endsWith('.gguf');
|
|
62
49
|
}
|
|
63
50
|
|
|
64
|
-
function
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
const basename = path.basename(sourcePath, path.extname(sourcePath)) || sourceKind;
|
|
75
|
-
return resolveConvertedModelId({
|
|
76
|
-
explicitModelId: basename,
|
|
77
|
-
converterConfig: null,
|
|
78
|
-
detectedModelId: basename,
|
|
79
|
-
quantizationInfo: plan.quantizationInfo,
|
|
80
|
-
}) || basename;
|
|
51
|
+
function isTflitePath(filePath) {
|
|
52
|
+
return String(filePath || '').toLowerCase().endsWith('.tflite');
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function isLiteRTTaskPath(filePath) {
|
|
56
|
+
return String(filePath || '').toLowerCase().endsWith('.task');
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function isLiteRTLMPath(filePath) {
|
|
60
|
+
return String(filePath || '').toLowerCase().endsWith('.litertlm');
|
|
81
61
|
}
|
|
82
62
|
|
|
83
63
|
async function getPathStats(targetPath, label) {
|
|
@@ -115,16 +95,6 @@ async function readJson(filePath, label) {
|
|
|
115
95
|
}
|
|
116
96
|
}
|
|
117
97
|
|
|
118
|
-
async function readFileBytes(filePath, label) {
|
|
119
|
-
try {
|
|
120
|
-
const bytes = await fs.readFile(filePath);
|
|
121
|
-
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
122
|
-
} catch (error) {
|
|
123
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
124
|
-
throw new Error(`Failed to read ${label} "${filePath}": ${message}`);
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
|
|
128
98
|
async function readRange(filePath, offset, length) {
|
|
129
99
|
if (!Number.isFinite(offset) || !Number.isFinite(length) || length <= 0) {
|
|
130
100
|
return new ArrayBuffer(0);
|
|
@@ -140,7 +110,8 @@ async function readRange(filePath, offset, length) {
|
|
|
140
110
|
const out = Buffer.allocUnsafe(end - start);
|
|
141
111
|
let pos = 0;
|
|
142
112
|
while (pos < out.length) {
|
|
143
|
-
const
|
|
113
|
+
const nextChunkBytes = Math.min(out.length - pos, MAX_NODE_READ_BYTES);
|
|
114
|
+
const { bytesRead } = await handle.read(out, pos, nextChunkBytes, start + pos);
|
|
144
115
|
if (bytesRead === 0) break;
|
|
145
116
|
pos += bytesRead;
|
|
146
117
|
}
|
|
@@ -166,6 +137,106 @@ async function readSafetensorsHeaderFromFile(filePath) {
|
|
|
166
137
|
);
|
|
167
138
|
}
|
|
168
139
|
|
|
140
|
+
async function collectTokenizerAssets(inputDir) {
|
|
141
|
+
const tokenizerJsonPath = path.join(inputDir, 'tokenizer.json');
|
|
142
|
+
const tokenizerConfigPath = path.join(inputDir, 'tokenizer_config.json');
|
|
143
|
+
const tokenizerModelPath = path.join(inputDir, 'tokenizer.model');
|
|
144
|
+
const tokenizerJson = await fileExists(tokenizerJsonPath)
|
|
145
|
+
? await readJson(tokenizerJsonPath, 'tokenizer.json')
|
|
146
|
+
: null;
|
|
147
|
+
const tokenizerConfig = await fileExists(tokenizerConfigPath)
|
|
148
|
+
? await readJson(tokenizerConfigPath, 'tokenizer_config.json')
|
|
149
|
+
: null;
|
|
150
|
+
const hasTokenizerModel = await fileExists(tokenizerModelPath);
|
|
151
|
+
return {
|
|
152
|
+
tokenizerJson,
|
|
153
|
+
tokenizerConfig,
|
|
154
|
+
tokenizerModelName: hasTokenizerModel ? 'tokenizer.model' : null,
|
|
155
|
+
tokenizerJsonPath,
|
|
156
|
+
tokenizerConfigPath,
|
|
157
|
+
tokenizerModelPath: hasTokenizerModel ? tokenizerModelPath : null,
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
async function buildTokenizerAuxiliaryFiles(tokenizerAssets) {
|
|
162
|
+
const auxiliaryFiles = [];
|
|
163
|
+
if (tokenizerAssets.tokenizerJson) {
|
|
164
|
+
auxiliaryFiles.push({
|
|
165
|
+
path: tokenizerAssets.tokenizerJsonPath,
|
|
166
|
+
size: Number((await getPathStats(tokenizerAssets.tokenizerJsonPath, 'tokenizer.json')).size),
|
|
167
|
+
kind: 'tokenizer_json',
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
if (tokenizerAssets.tokenizerConfig) {
|
|
171
|
+
auxiliaryFiles.push({
|
|
172
|
+
path: tokenizerAssets.tokenizerConfigPath,
|
|
173
|
+
size: Number((await getPathStats(tokenizerAssets.tokenizerConfigPath, 'tokenizer_config.json')).size),
|
|
174
|
+
kind: 'tokenizer_config',
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
if (tokenizerAssets.tokenizerModelPath) {
|
|
178
|
+
auxiliaryFiles.push({
|
|
179
|
+
path: tokenizerAssets.tokenizerModelPath,
|
|
180
|
+
size: Number((await getPathStats(tokenizerAssets.tokenizerModelPath, 'tokenizer.model')).size),
|
|
181
|
+
kind: 'tokenizer_model',
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
return auxiliaryFiles;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function buildPackageTokenizerVirtualFiles(tokenizerAssets) {
|
|
188
|
+
const virtualFiles = [];
|
|
189
|
+
if (tokenizerAssets.tokenizerJson) {
|
|
190
|
+
virtualFiles.push({
|
|
191
|
+
path: 'tokenizer.json',
|
|
192
|
+
offset: 0,
|
|
193
|
+
size: 0,
|
|
194
|
+
kind: 'tokenizer_json',
|
|
195
|
+
externalPath: tokenizerAssets.tokenizerJsonPath,
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
if (tokenizerAssets.tokenizerConfig) {
|
|
199
|
+
virtualFiles.push({
|
|
200
|
+
path: 'tokenizer_config.json',
|
|
201
|
+
offset: 0,
|
|
202
|
+
size: 0,
|
|
203
|
+
kind: 'tokenizer_config',
|
|
204
|
+
externalPath: tokenizerAssets.tokenizerConfigPath,
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
if (tokenizerAssets.tokenizerModelPath) {
|
|
208
|
+
virtualFiles.push({
|
|
209
|
+
path: 'TOKENIZER_MODEL',
|
|
210
|
+
offset: 0,
|
|
211
|
+
size: 0,
|
|
212
|
+
kind: 'tokenizer_model',
|
|
213
|
+
externalPath: tokenizerAssets.tokenizerModelPath,
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
return virtualFiles;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function applyPackageTokenizerAssets(parsedArtifact, tokenizerAssets) {
|
|
220
|
+
const next = {
|
|
221
|
+
...parsedArtifact,
|
|
222
|
+
};
|
|
223
|
+
if (tokenizerAssets.tokenizerJson && next.tokenizerJson == null) {
|
|
224
|
+
next.tokenizerJson = tokenizerAssets.tokenizerJson;
|
|
225
|
+
next.tokenizerJsonPath = 'tokenizer.json';
|
|
226
|
+
}
|
|
227
|
+
if (tokenizerAssets.tokenizerConfig && next.tokenizerConfigPath == null) {
|
|
228
|
+
next.tokenizerConfigPath = 'tokenizer_config.json';
|
|
229
|
+
}
|
|
230
|
+
if (tokenizerAssets.tokenizerConfig && next.tokenizerConfig == null) {
|
|
231
|
+
next.tokenizerConfig = tokenizerAssets.tokenizerConfig;
|
|
232
|
+
}
|
|
233
|
+
if (tokenizerAssets.tokenizerModelPath && next.tokenizerModelPath == null) {
|
|
234
|
+
next.tokenizerModelName = 'TOKENIZER_MODEL';
|
|
235
|
+
next.tokenizerModelPath = 'TOKENIZER_MODEL';
|
|
236
|
+
}
|
|
237
|
+
return next;
|
|
238
|
+
}
|
|
239
|
+
|
|
169
240
|
async function parseSafetensorsInput(inputDir) {
|
|
170
241
|
const configPath = path.join(inputDir, 'config.json');
|
|
171
242
|
if (!(await fileExists(configPath))) {
|
|
@@ -214,17 +285,7 @@ async function parseSafetensorsInput(inputDir) {
|
|
|
214
285
|
const architectureHint = parsedTransformer.architectureHint;
|
|
215
286
|
const embeddingPostprocessor = parsedTransformer.embeddingPostprocessor ?? null;
|
|
216
287
|
const architecture = extractArchitecture(config, null);
|
|
217
|
-
|
|
218
|
-
const tokenizerJsonPath = path.join(inputDir, 'tokenizer.json');
|
|
219
|
-
const tokenizerConfigPath = path.join(inputDir, 'tokenizer_config.json');
|
|
220
|
-
const tokenizerModelPath = path.join(inputDir, 'tokenizer.model');
|
|
221
|
-
const tokenizerJson = await fileExists(tokenizerJsonPath)
|
|
222
|
-
? await readJson(tokenizerJsonPath, 'tokenizer.json')
|
|
223
|
-
: null;
|
|
224
|
-
const tokenizerConfig = await fileExists(tokenizerConfigPath)
|
|
225
|
-
? await readJson(tokenizerConfigPath, 'tokenizer_config.json')
|
|
226
|
-
: null;
|
|
227
|
-
const hasTokenizerModel = await fileExists(tokenizerModelPath);
|
|
288
|
+
const tokenizerAssets = await collectTokenizerAssets(inputDir);
|
|
228
289
|
|
|
229
290
|
const sourceFiles = [];
|
|
230
291
|
const uniquePaths = new Set(tensors.map((tensor) => normalizePath(tensor.sourcePath)));
|
|
@@ -241,27 +302,7 @@ async function parseSafetensorsInput(inputDir) {
|
|
|
241
302
|
kind: 'safetensors_index',
|
|
242
303
|
}]
|
|
243
304
|
: []),
|
|
244
|
-
...(
|
|
245
|
-
? [{
|
|
246
|
-
path: tokenizerJsonPath,
|
|
247
|
-
size: Number((await getPathStats(tokenizerJsonPath, 'tokenizer.json')).size),
|
|
248
|
-
kind: 'tokenizer_json',
|
|
249
|
-
}]
|
|
250
|
-
: []),
|
|
251
|
-
...(tokenizerConfig
|
|
252
|
-
? [{
|
|
253
|
-
path: tokenizerConfigPath,
|
|
254
|
-
size: Number((await getPathStats(tokenizerConfigPath, 'tokenizer_config.json')).size),
|
|
255
|
-
kind: 'tokenizer_config',
|
|
256
|
-
}]
|
|
257
|
-
: []),
|
|
258
|
-
...(hasTokenizerModel
|
|
259
|
-
? [{
|
|
260
|
-
path: tokenizerModelPath,
|
|
261
|
-
size: Number((await getPathStats(tokenizerModelPath, 'tokenizer.model')).size),
|
|
262
|
-
kind: 'tokenizer_model',
|
|
263
|
-
}]
|
|
264
|
-
: []),
|
|
305
|
+
...await buildTokenizerAuxiliaryFiles(tokenizerAssets),
|
|
265
306
|
];
|
|
266
307
|
|
|
267
308
|
return {
|
|
@@ -273,18 +314,226 @@ async function parseSafetensorsInput(inputDir) {
|
|
|
273
314
|
architectureHint,
|
|
274
315
|
embeddingPostprocessor,
|
|
275
316
|
architecture,
|
|
276
|
-
sourceQuantization: inferSourceQuantizationForSourceRuntime(tensors, 'safetensors'
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
317
|
+
sourceQuantization: inferSourceQuantizationForSourceRuntime(tensors, 'safetensors', {
|
|
318
|
+
logCategory: 'NodeSourceRuntime',
|
|
319
|
+
}),
|
|
320
|
+
tokenizerJson: tokenizerAssets.tokenizerJson,
|
|
321
|
+
tokenizerConfig: tokenizerAssets.tokenizerConfig,
|
|
322
|
+
tokenizerModelName: tokenizerAssets.tokenizerModelName,
|
|
323
|
+
tokenizerJsonPath: tokenizerAssets.tokenizerJsonPath,
|
|
324
|
+
tokenizerConfigPath: tokenizerAssets.tokenizerConfigPath,
|
|
325
|
+
tokenizerModelPath: tokenizerAssets.tokenizerModelPath,
|
|
283
326
|
sourceFiles,
|
|
284
327
|
auxiliaryFiles,
|
|
285
328
|
};
|
|
286
329
|
}
|
|
287
330
|
|
|
331
|
+
async function parseTfliteInput(tflitePath) {
|
|
332
|
+
const tfliteStats = await getPathStats(tflitePath, 'TFLite file');
|
|
333
|
+
const inputDir = path.dirname(tflitePath);
|
|
334
|
+
const configPath = path.join(inputDir, 'config.json');
|
|
335
|
+
if (!(await fileExists(configPath))) {
|
|
336
|
+
throw new Error(
|
|
337
|
+
`node source runtime: config.json is required next to TFLite source "${tflitePath}".`
|
|
338
|
+
);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
const config = await readJson(configPath, 'config.json');
|
|
342
|
+
const parsedTFLite = await parseTFLiteFromSource({
|
|
343
|
+
name: path.basename(tflitePath),
|
|
344
|
+
size: Number(tfliteStats.size),
|
|
345
|
+
async readRange(offset, length) {
|
|
346
|
+
return readRange(tflitePath, offset, length);
|
|
347
|
+
},
|
|
348
|
+
});
|
|
349
|
+
const tokenizerAssets = await collectTokenizerAssets(inputDir);
|
|
350
|
+
const tensors = parsedTFLite.tensors.map((tensor) => ({
|
|
351
|
+
...tensor,
|
|
352
|
+
sourcePath: tflitePath,
|
|
353
|
+
}));
|
|
354
|
+
const architecture = extractArchitecture(config, null);
|
|
355
|
+
const architectureHint = config.architectures?.[0] ?? config.model_type ?? '';
|
|
356
|
+
const auxiliaryFiles = [
|
|
357
|
+
{ path: configPath, size: Number((await getPathStats(configPath, 'config.json')).size), kind: 'config' },
|
|
358
|
+
...await buildTokenizerAuxiliaryFiles(tokenizerAssets),
|
|
359
|
+
];
|
|
360
|
+
|
|
361
|
+
return {
|
|
362
|
+
sourceKind: 'tflite',
|
|
363
|
+
sourceRoot: inputDir,
|
|
364
|
+
sourcePathForModelId: tflitePath,
|
|
365
|
+
config,
|
|
366
|
+
tensors,
|
|
367
|
+
architectureHint,
|
|
368
|
+
embeddingPostprocessor: null,
|
|
369
|
+
architecture,
|
|
370
|
+
sourceQuantization: parsedTFLite.sourceQuantization,
|
|
371
|
+
tokenizerJson: tokenizerAssets.tokenizerJson,
|
|
372
|
+
tokenizerConfig: tokenizerAssets.tokenizerConfig,
|
|
373
|
+
tokenizerModelName: tokenizerAssets.tokenizerModelName,
|
|
374
|
+
tokenizerJsonPath: tokenizerAssets.tokenizerJsonPath,
|
|
375
|
+
tokenizerConfigPath: tokenizerAssets.tokenizerConfigPath,
|
|
376
|
+
tokenizerModelPath: tokenizerAssets.tokenizerModelPath,
|
|
377
|
+
sourceFiles: [{ path: tflitePath, size: Number(tfliteStats.size) }],
|
|
378
|
+
auxiliaryFiles,
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
function createNodeVirtualFileReaders(packagePath, virtualFiles) {
|
|
383
|
+
const virtualFileMap = new Map(
|
|
384
|
+
(Array.isArray(virtualFiles) ? virtualFiles : []).map((entry) => [entry.path, entry])
|
|
385
|
+
);
|
|
386
|
+
|
|
387
|
+
const resolveVirtualFile = (virtualPath) => {
|
|
388
|
+
const normalized = normalizePath(virtualPath);
|
|
389
|
+
const entry = virtualFileMap.get(normalized);
|
|
390
|
+
if (!entry) {
|
|
391
|
+
throw new Error(`node source runtime: missing package asset "${virtualPath}".`);
|
|
392
|
+
}
|
|
393
|
+
return entry;
|
|
394
|
+
};
|
|
395
|
+
|
|
396
|
+
const readRangeFromVirtualFile = async (virtualPath, offset, length) => {
|
|
397
|
+
const entry = resolveVirtualFile(virtualPath);
|
|
398
|
+
if (entry.externalPath) {
|
|
399
|
+
return readRange(entry.externalPath, offset, length);
|
|
400
|
+
}
|
|
401
|
+
const start = Math.max(0, Math.floor(Number(offset) || 0));
|
|
402
|
+
const requested = Math.max(0, Math.floor(Number(length) || 0));
|
|
403
|
+
const available = Math.max(0, entry.size - start);
|
|
404
|
+
const readLength = Math.min(available, requested);
|
|
405
|
+
if (readLength <= 0) {
|
|
406
|
+
return new ArrayBuffer(0);
|
|
407
|
+
}
|
|
408
|
+
return readRange(packagePath, entry.offset + start, readLength);
|
|
409
|
+
};
|
|
410
|
+
|
|
411
|
+
const streamRange = async function* (virtualPath, offset, length, options = {}) {
|
|
412
|
+
const entry = resolveVirtualFile(virtualPath);
|
|
413
|
+
if (entry.externalPath) {
|
|
414
|
+
const stats = await getPathStats(entry.externalPath, `package sidecar (${entry.externalPath})`);
|
|
415
|
+
const start = Math.max(0, Math.floor(Number(offset) || 0));
|
|
416
|
+
const requested = Math.max(0, Math.floor(Number(length) || 0));
|
|
417
|
+
const available = Math.max(0, Number(stats.size) - start);
|
|
418
|
+
const readLength = Math.min(available, requested);
|
|
419
|
+
if (readLength <= 0) {
|
|
420
|
+
return;
|
|
421
|
+
}
|
|
422
|
+
const chunkBytesRaw = Number(options?.chunkBytes);
|
|
423
|
+
const highWaterMark = Number.isFinite(chunkBytesRaw) && chunkBytesRaw > 0
|
|
424
|
+
? Math.floor(chunkBytesRaw)
|
|
425
|
+
: MAX_NODE_READ_BYTES;
|
|
426
|
+
const stream = createReadStream(entry.externalPath, {
|
|
427
|
+
start,
|
|
428
|
+
end: start + readLength - 1,
|
|
429
|
+
highWaterMark,
|
|
430
|
+
});
|
|
431
|
+
for await (const chunk of stream) {
|
|
432
|
+
yield chunk;
|
|
433
|
+
}
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
const start = Math.max(0, Math.floor(Number(offset) || 0));
|
|
437
|
+
const requested = Math.max(0, Math.floor(Number(length) || 0));
|
|
438
|
+
const available = Math.max(0, entry.size - start);
|
|
439
|
+
const readLength = Math.min(available, requested);
|
|
440
|
+
if (readLength <= 0) {
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
443
|
+
const chunkBytesRaw = Number(options?.chunkBytes);
|
|
444
|
+
const highWaterMark = Number.isFinite(chunkBytesRaw) && chunkBytesRaw > 0
|
|
445
|
+
? Math.floor(chunkBytesRaw)
|
|
446
|
+
: MAX_NODE_READ_BYTES;
|
|
447
|
+
const stream = createReadStream(packagePath, {
|
|
448
|
+
start: entry.offset + start,
|
|
449
|
+
end: entry.offset + start + readLength - 1,
|
|
450
|
+
highWaterMark,
|
|
451
|
+
});
|
|
452
|
+
for await (const chunk of stream) {
|
|
453
|
+
yield chunk;
|
|
454
|
+
}
|
|
455
|
+
};
|
|
456
|
+
|
|
457
|
+
const readText = async (virtualPath) => {
|
|
458
|
+
const entry = resolveVirtualFile(virtualPath);
|
|
459
|
+
const bytes = entry.externalPath
|
|
460
|
+
? await fs.readFile(entry.externalPath)
|
|
461
|
+
: await readRange(packagePath, entry.offset, entry.size);
|
|
462
|
+
return new TextDecoder().decode(bytes);
|
|
463
|
+
};
|
|
464
|
+
|
|
465
|
+
const readBinary = async (virtualPath) => {
|
|
466
|
+
const entry = resolveVirtualFile(virtualPath);
|
|
467
|
+
return entry.externalPath
|
|
468
|
+
? fs.readFile(entry.externalPath)
|
|
469
|
+
: readRange(packagePath, entry.offset, entry.size);
|
|
470
|
+
};
|
|
471
|
+
|
|
472
|
+
return {
|
|
473
|
+
virtualFileMap,
|
|
474
|
+
readRange: readRangeFromVirtualFile,
|
|
475
|
+
streamRange,
|
|
476
|
+
readText,
|
|
477
|
+
readBinary,
|
|
478
|
+
};
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
async function addHashesToVirtualEntries(packagePath, virtualFiles, entries, hashAlgorithm) {
|
|
482
|
+
const readers = createNodeVirtualFileReaders(packagePath, virtualFiles);
|
|
483
|
+
const hashedEntries = [];
|
|
484
|
+
for (const entry of Array.isArray(entries) ? entries : []) {
|
|
485
|
+
const virtualPath = normalizePath(entry?.path);
|
|
486
|
+
if (!virtualPath) {
|
|
487
|
+
continue;
|
|
488
|
+
}
|
|
489
|
+
const descriptor = readers.virtualFileMap.get(virtualPath);
|
|
490
|
+
if (!descriptor) {
|
|
491
|
+
throw new Error(`node source runtime: missing virtual package asset "${virtualPath}" for hashing.`);
|
|
492
|
+
}
|
|
493
|
+
const hash = createHash(hashAlgorithm);
|
|
494
|
+
for await (const chunk of readers.streamRange(virtualPath, 0, descriptor.size)) {
|
|
495
|
+
hash.update(chunk);
|
|
496
|
+
}
|
|
497
|
+
hashedEntries.push({
|
|
498
|
+
...entry,
|
|
499
|
+
path: virtualPath,
|
|
500
|
+
size: Number.isFinite(entry?.size) ? Math.max(0, Math.floor(Number(entry.size))) : descriptor.size,
|
|
501
|
+
hash: hash.digest('hex'),
|
|
502
|
+
hashAlgorithm,
|
|
503
|
+
});
|
|
504
|
+
}
|
|
505
|
+
return hashedEntries;
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
async function parseLiteRTPackageInput(packagePath, sourceKind) {
|
|
509
|
+
const stats = await getPathStats(packagePath, `LiteRT package (${sourceKind})`);
|
|
510
|
+
const resolved = await resolveLiteRTPackageParsedArtifact({
|
|
511
|
+
sourceKind,
|
|
512
|
+
sourcePathForModelId: packagePath,
|
|
513
|
+
source: {
|
|
514
|
+
name: path.basename(packagePath),
|
|
515
|
+
size: Number(stats.size),
|
|
516
|
+
async readRange(offset, length) {
|
|
517
|
+
return readRange(packagePath, offset, length);
|
|
518
|
+
},
|
|
519
|
+
},
|
|
520
|
+
});
|
|
521
|
+
const tokenizerAssets = await collectTokenizerAssets(path.dirname(packagePath));
|
|
522
|
+
const virtualFiles = appendLiteRTPackageVirtualFiles(
|
|
523
|
+
resolved.virtualFiles,
|
|
524
|
+
buildPackageTokenizerVirtualFiles(tokenizerAssets)
|
|
525
|
+
);
|
|
526
|
+
const parsedArtifact = applyPackageTokenizerAssets(resolved.parsedArtifact, tokenizerAssets);
|
|
527
|
+
return {
|
|
528
|
+
...parsedArtifact,
|
|
529
|
+
sourceRoot: packagePath,
|
|
530
|
+
storageReaders: createNodeVirtualFileReaders(packagePath, virtualFiles),
|
|
531
|
+
async hashFileEntries(entries, hashAlgorithm) {
|
|
532
|
+
return addHashesToVirtualEntries(packagePath, virtualFiles, entries, hashAlgorithm);
|
|
533
|
+
},
|
|
534
|
+
};
|
|
535
|
+
}
|
|
536
|
+
|
|
288
537
|
async function parseGgufInput(ggufPath) {
|
|
289
538
|
const ggufStats = await getPathStats(ggufPath, 'GGUF file');
|
|
290
539
|
const fileSize = Number(ggufStats.size);
|
|
@@ -338,7 +587,9 @@ async function parseGgufInput(ggufPath) {
|
|
|
338
587
|
tensors,
|
|
339
588
|
architectureHint: parsed.architecture,
|
|
340
589
|
architecture: extractArchitecture({}, parsed.config || {}),
|
|
341
|
-
sourceQuantization: parsed.quantization ?? inferSourceQuantizationForSourceRuntime(tensors, 'gguf'
|
|
590
|
+
sourceQuantization: parsed.quantization ?? inferSourceQuantizationForSourceRuntime(tensors, 'gguf', {
|
|
591
|
+
logCategory: 'NodeSourceRuntime',
|
|
592
|
+
}),
|
|
342
593
|
tokenizerJson: null,
|
|
343
594
|
tokenizerConfig: null,
|
|
344
595
|
tokenizerModelName: null,
|
|
@@ -350,54 +601,31 @@ async function parseGgufInput(ggufPath) {
|
|
|
350
601
|
};
|
|
351
602
|
}
|
|
352
603
|
|
|
353
|
-
function
|
|
354
|
-
const
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
unsupported.add('(empty)');
|
|
359
|
-
continue;
|
|
360
|
-
}
|
|
361
|
-
if (!SUPPORTED_SOURCE_DTYPES.has(dtype)) {
|
|
362
|
-
unsupported.add(dtype);
|
|
604
|
+
function buildNodeFileReaders() {
|
|
605
|
+
const readRangeFromFile = async (filePath, offset, length) => readRange(filePath, offset, length);
|
|
606
|
+
const streamRange = async function* (filePath, offset, length, options = {}) {
|
|
607
|
+
if (!Number.isFinite(offset) || !Number.isFinite(length) || length <= 0) {
|
|
608
|
+
return;
|
|
363
609
|
}
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
'Convert to RDRR first for this model.'
|
|
370
|
-
);
|
|
371
|
-
}
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
function inferSourceQuantizationForSourceRuntime(tensors, sourceKind) {
|
|
375
|
-
try {
|
|
376
|
-
return inferSourceWeightQuantization(tensors);
|
|
377
|
-
} catch (error) {
|
|
378
|
-
const dtypes = new Set();
|
|
379
|
-
for (const tensor of tensors) {
|
|
380
|
-
const dtype = String(tensor?.dtype || '').trim().toUpperCase();
|
|
381
|
-
if (dtype) dtypes.add(dtype);
|
|
610
|
+
const stats = await getPathStats(filePath, `stream source asset (${filePath})`);
|
|
611
|
+
const start = Math.max(0, Math.floor(offset));
|
|
612
|
+
const end = Math.min(Number(stats.size), start + Math.floor(length));
|
|
613
|
+
if (end <= start) {
|
|
614
|
+
return;
|
|
382
615
|
}
|
|
383
|
-
const
|
|
384
|
-
const
|
|
385
|
-
(
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
616
|
+
const chunkBytesRaw = Number(options?.chunkBytes);
|
|
617
|
+
const highWaterMark = Number.isFinite(chunkBytesRaw) && chunkBytesRaw > 0
|
|
618
|
+
? Math.floor(chunkBytesRaw)
|
|
619
|
+
: MAX_NODE_READ_BYTES;
|
|
620
|
+
const stream = createReadStream(filePath, {
|
|
621
|
+
start,
|
|
622
|
+
end: end - 1,
|
|
623
|
+
highWaterMark,
|
|
624
|
+
});
|
|
625
|
+
for await (const chunk of stream) {
|
|
626
|
+
yield chunk;
|
|
394
627
|
}
|
|
395
|
-
|
|
396
|
-
}
|
|
397
|
-
}
|
|
398
|
-
|
|
399
|
-
function buildNodeFileReaders() {
|
|
400
|
-
const readRangeFromFile = async (filePath, offset, length) => readRange(filePath, offset, length);
|
|
628
|
+
};
|
|
401
629
|
const readText = async (filePath) => {
|
|
402
630
|
try {
|
|
403
631
|
return await fs.readFile(filePath, 'utf8');
|
|
@@ -414,44 +642,12 @@ function buildNodeFileReaders() {
|
|
|
414
642
|
};
|
|
415
643
|
return {
|
|
416
644
|
readRange: readRangeFromFile,
|
|
645
|
+
streamRange,
|
|
417
646
|
readText,
|
|
418
647
|
readBinary,
|
|
419
648
|
};
|
|
420
649
|
}
|
|
421
650
|
|
|
422
|
-
// Source dtype → compute precision mapping for source-runtime inference.
|
|
423
|
-
// BF16/F32 sources require f32 compute (BF16 has no native WebGPU support).
|
|
424
|
-
// Quantized formats require f32 compute for dequantization accuracy.
|
|
425
|
-
// F16 sources can use f16 compute directly.
|
|
426
|
-
const SOURCE_QUANT_COMPUTE_MAP = {
|
|
427
|
-
'F16': 'f16',
|
|
428
|
-
'BF16': 'f32',
|
|
429
|
-
'F32': 'f32',
|
|
430
|
-
'Q4_K': 'f32',
|
|
431
|
-
'Q4_K_M': 'f32',
|
|
432
|
-
'Q6_K': 'f32',
|
|
433
|
-
};
|
|
434
|
-
const SOURCE_COMPUTE_DEFAULT = 'f16';
|
|
435
|
-
|
|
436
|
-
function resolveSourceRuntimeComputePrecision(tensors, sourceQuantization) {
|
|
437
|
-
const dtypes = new Set();
|
|
438
|
-
for (const tensor of Array.isArray(tensors) ? tensors : []) {
|
|
439
|
-
const dtype = String(tensor?.dtype || '').trim().toUpperCase();
|
|
440
|
-
if (dtype) {
|
|
441
|
-
dtypes.add(dtype);
|
|
442
|
-
}
|
|
443
|
-
}
|
|
444
|
-
// If any tensor requires f32 compute, use f32 for all.
|
|
445
|
-
for (const dtype of dtypes) {
|
|
446
|
-
if (SOURCE_QUANT_COMPUTE_MAP[dtype] === 'f32') {
|
|
447
|
-
return 'f32';
|
|
448
|
-
}
|
|
449
|
-
}
|
|
450
|
-
|
|
451
|
-
const normalized = String(sourceQuantization || '').trim().toUpperCase();
|
|
452
|
-
return SOURCE_QUANT_COMPUTE_MAP[normalized] ?? SOURCE_COMPUTE_DEFAULT;
|
|
453
|
-
}
|
|
454
|
-
|
|
455
651
|
async function addHashesToFileEntries(entries, hashAlgorithm) {
|
|
456
652
|
const normalized = [];
|
|
457
653
|
for (const entry of Array.isArray(entries) ? entries : []) {
|
|
@@ -487,6 +683,115 @@ async function computeFileHash(filePath, hashAlgorithm) {
|
|
|
487
683
|
});
|
|
488
684
|
}
|
|
489
685
|
|
|
686
|
+
function resolveLoadingConfig(runtimeConfig) {
|
|
687
|
+
const loadingConfig = runtimeConfig?.loading ?? getRuntimeConfig().loading;
|
|
688
|
+
if (!loadingConfig || typeof loadingConfig !== 'object') {
|
|
689
|
+
throw new Error('node source runtime: runtime.loading is required.');
|
|
690
|
+
}
|
|
691
|
+
return loadingConfig;
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
function resolveMemoryBudgetConfig(loadingConfig) {
|
|
695
|
+
const budget = loadingConfig?.memoryManagement?.budget;
|
|
696
|
+
if (!budget || typeof budget !== 'object') {
|
|
697
|
+
throw new Error('node source runtime: runtime.loading.memoryManagement.budget is required.');
|
|
698
|
+
}
|
|
699
|
+
return budget;
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
function resolveResidentBudgetBytes(loadingConfig) {
|
|
703
|
+
const budget = resolveMemoryBudgetConfig(loadingConfig);
|
|
704
|
+
if (budget.enabled !== true) {
|
|
705
|
+
return null;
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
const explicitMaxResidentBytes = budget.maxResidentBytes;
|
|
709
|
+
if (explicitMaxResidentBytes != null) {
|
|
710
|
+
const normalized = Number(explicitMaxResidentBytes);
|
|
711
|
+
if (!Number.isFinite(normalized) || normalized <= 0) {
|
|
712
|
+
throw new Error(
|
|
713
|
+
'node source runtime: runtime.loading.memoryManagement.budget.maxResidentBytes ' +
|
|
714
|
+
'must be a positive number or null.'
|
|
715
|
+
);
|
|
716
|
+
}
|
|
717
|
+
return Math.floor(normalized);
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
const systemMemoryFraction = Number(budget.systemMemoryFraction);
|
|
721
|
+
const reserveBytes = Number(budget.reserveBytes);
|
|
722
|
+
const minimumBudgetBytes = Number(budget.minimumBudgetBytes);
|
|
723
|
+
if (!Number.isFinite(systemMemoryFraction) || systemMemoryFraction <= 0 || systemMemoryFraction > 1) {
|
|
724
|
+
throw new Error(
|
|
725
|
+
'node source runtime: runtime.loading.memoryManagement.budget.systemMemoryFraction ' +
|
|
726
|
+
'must be within (0, 1].'
|
|
727
|
+
);
|
|
728
|
+
}
|
|
729
|
+
if (!Number.isFinite(reserveBytes) || reserveBytes < 0) {
|
|
730
|
+
throw new Error(
|
|
731
|
+
'node source runtime: runtime.loading.memoryManagement.budget.reserveBytes ' +
|
|
732
|
+
'must be a non-negative number.'
|
|
733
|
+
);
|
|
734
|
+
}
|
|
735
|
+
if (!Number.isFinite(minimumBudgetBytes) || minimumBudgetBytes <= 0) {
|
|
736
|
+
throw new Error(
|
|
737
|
+
'node source runtime: runtime.loading.memoryManagement.budget.minimumBudgetBytes ' +
|
|
738
|
+
'must be a positive number.'
|
|
739
|
+
);
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
const derived = Math.floor(totalmem() * systemMemoryFraction) - Math.floor(reserveBytes);
|
|
743
|
+
return Math.max(Math.floor(minimumBudgetBytes), derived);
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
function estimateSourceRuntimeTransientBytes(parsed, loadingConfig) {
|
|
747
|
+
const maxTensorBytes = (Array.isArray(parsed?.tensors) ? parsed.tensors : []).reduce((maxBytes, tensor) => {
|
|
748
|
+
const size = Number(tensor?.size);
|
|
749
|
+
return Number.isFinite(size) && size > maxBytes ? Math.floor(size) : maxBytes;
|
|
750
|
+
}, 0);
|
|
751
|
+
const streamConfig = loadingConfig?.storage?.backend?.streaming ?? {};
|
|
752
|
+
const readChunkBytes = Number(streamConfig.readChunkBytes);
|
|
753
|
+
const maxInFlightBytes = Number(streamConfig.maxInFlightBytes);
|
|
754
|
+
const streamWindowBytes = Math.max(
|
|
755
|
+
Number.isFinite(readChunkBytes) && readChunkBytes > 0 ? Math.floor(readChunkBytes) : 0,
|
|
756
|
+
Number.isFinite(maxInFlightBytes) && maxInFlightBytes > 0 ? Math.floor(maxInFlightBytes) : 0,
|
|
757
|
+
MAX_NODE_READ_BYTES
|
|
758
|
+
);
|
|
759
|
+
return Math.max(maxTensorBytes, streamWindowBytes);
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
function assertSourceRuntimeFitsResidentBudget(parsed, loadingConfig, residentBudgetBytes) {
|
|
763
|
+
if (!Number.isFinite(residentBudgetBytes) || residentBudgetBytes <= 0) {
|
|
764
|
+
return;
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
const currentRssBytes = typeof process !== 'undefined' && typeof process.memoryUsage === 'function'
|
|
768
|
+
? process.memoryUsage().rss
|
|
769
|
+
: 0;
|
|
770
|
+
const estimatedTransientBytes = estimateSourceRuntimeTransientBytes(parsed, loadingConfig);
|
|
771
|
+
const projectedResidentBytes = currentRssBytes + estimatedTransientBytes;
|
|
772
|
+
if (projectedResidentBytes <= residentBudgetBytes) {
|
|
773
|
+
return;
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
const sourceFiles = Array.isArray(parsed?.sourceFiles) ? parsed.sourceFiles : [];
|
|
777
|
+
const totalSourceBytes = sourceFiles.reduce((totalBytes, entry) => {
|
|
778
|
+
const size = Number(entry?.size);
|
|
779
|
+
return Number.isFinite(size) ? totalBytes + Math.max(0, Math.floor(size)) : totalBytes;
|
|
780
|
+
}, 0);
|
|
781
|
+
const largestSourceBytes = sourceFiles.reduce((maxBytes, entry) => {
|
|
782
|
+
const size = Number(entry?.size);
|
|
783
|
+
return Number.isFinite(size) && size > maxBytes ? Math.floor(size) : maxBytes;
|
|
784
|
+
}, 0);
|
|
785
|
+
|
|
786
|
+
throw new Error(
|
|
787
|
+
'node source runtime: direct-source load exceeds resident memory budget. ' +
|
|
788
|
+
`rss=${formatBytes(currentRssBytes)}, transient=${formatBytes(estimatedTransientBytes)}, ` +
|
|
789
|
+
`projected=${formatBytes(projectedResidentBytes)}, budget=${formatBytes(residentBudgetBytes)}, ` +
|
|
790
|
+
`largestSource=${formatBytes(largestSourceBytes)}, totalSource=${formatBytes(totalSourceBytes)}. ` +
|
|
791
|
+
'Lower the model working set or adjust runtime.loading.memoryManagement.budget.'
|
|
792
|
+
);
|
|
793
|
+
}
|
|
794
|
+
|
|
490
795
|
export async function resolveNodeSourceRuntimeBundle(options = {}) {
|
|
491
796
|
const inputPath = normalizePath(options.inputPath);
|
|
492
797
|
if (!inputPath) {
|
|
@@ -498,10 +803,21 @@ export async function resolveNodeSourceRuntimeBundle(options = {}) {
|
|
|
498
803
|
|
|
499
804
|
let parsed = null;
|
|
500
805
|
if (stats.isFile()) {
|
|
501
|
-
if (
|
|
806
|
+
if (isTflitePath(resolvedInputPath)) {
|
|
807
|
+
parsed = await parseTfliteInput(resolvedInputPath);
|
|
808
|
+
}
|
|
809
|
+
if (!parsed && isLiteRTTaskPath(resolvedInputPath)) {
|
|
810
|
+
parsed = await parseLiteRTPackageInput(resolvedInputPath, LITERT_PACKAGE_SOURCE_KIND_TASK);
|
|
811
|
+
}
|
|
812
|
+
if (!parsed && isLiteRTLMPath(resolvedInputPath)) {
|
|
813
|
+
parsed = await parseLiteRTPackageInput(resolvedInputPath, LITERT_PACKAGE_SOURCE_KIND_LITERTLM);
|
|
814
|
+
}
|
|
815
|
+
if (!parsed && !isGgufPath(resolvedInputPath)) {
|
|
502
816
|
return null;
|
|
503
817
|
}
|
|
504
|
-
|
|
818
|
+
if (!parsed) {
|
|
819
|
+
parsed = await parseGgufInput(resolvedInputPath);
|
|
820
|
+
}
|
|
505
821
|
} else if (stats.isDirectory()) {
|
|
506
822
|
if (await fileExists(path.join(resolvedInputPath, 'manifest.json'))) {
|
|
507
823
|
return null;
|
|
@@ -520,6 +836,51 @@ export async function resolveNodeSourceRuntimeBundle(options = {}) {
|
|
|
520
836
|
`node source runtime: multiple GGUF files found in "${resolvedInputPath}": ${ggufFiles.join(', ')}.`
|
|
521
837
|
);
|
|
522
838
|
}
|
|
839
|
+
if (!parsed) {
|
|
840
|
+
const tfliteFiles = entries
|
|
841
|
+
.filter((entry) => entry.isFile() && isTflitePath(entry.name))
|
|
842
|
+
.map((entry) => entry.name)
|
|
843
|
+
.sort((left, right) => left.localeCompare(right));
|
|
844
|
+
if (tfliteFiles.length === 1) {
|
|
845
|
+
parsed = await parseTfliteInput(path.join(resolvedInputPath, tfliteFiles[0]));
|
|
846
|
+
} else if (tfliteFiles.length > 1) {
|
|
847
|
+
throw new Error(
|
|
848
|
+
`node source runtime: multiple TFLite files found in "${resolvedInputPath}": ${tfliteFiles.join(', ')}.`
|
|
849
|
+
);
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
if (!parsed) {
|
|
853
|
+
const taskFiles = entries
|
|
854
|
+
.filter((entry) => entry.isFile() && isLiteRTTaskPath(entry.name))
|
|
855
|
+
.map((entry) => entry.name)
|
|
856
|
+
.sort((left, right) => left.localeCompare(right));
|
|
857
|
+
if (taskFiles.length === 1) {
|
|
858
|
+
parsed = await parseLiteRTPackageInput(
|
|
859
|
+
path.join(resolvedInputPath, taskFiles[0]),
|
|
860
|
+
LITERT_PACKAGE_SOURCE_KIND_TASK
|
|
861
|
+
);
|
|
862
|
+
} else if (taskFiles.length > 1) {
|
|
863
|
+
throw new Error(
|
|
864
|
+
`node source runtime: multiple LiteRT task files found in "${resolvedInputPath}": ${taskFiles.join(', ')}.`
|
|
865
|
+
);
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
if (!parsed) {
|
|
869
|
+
const litertLmFiles = entries
|
|
870
|
+
.filter((entry) => entry.isFile() && isLiteRTLMPath(entry.name))
|
|
871
|
+
.map((entry) => entry.name)
|
|
872
|
+
.sort((left, right) => left.localeCompare(right));
|
|
873
|
+
if (litertLmFiles.length === 1) {
|
|
874
|
+
parsed = await parseLiteRTPackageInput(
|
|
875
|
+
path.join(resolvedInputPath, litertLmFiles[0]),
|
|
876
|
+
LITERT_PACKAGE_SOURCE_KIND_LITERTLM
|
|
877
|
+
);
|
|
878
|
+
} else if (litertLmFiles.length > 1) {
|
|
879
|
+
throw new Error(
|
|
880
|
+
`node source runtime: multiple LiteRT-LM files found in "${resolvedInputPath}": ${litertLmFiles.join(', ')}.`
|
|
881
|
+
);
|
|
882
|
+
}
|
|
883
|
+
}
|
|
523
884
|
}
|
|
524
885
|
} else {
|
|
525
886
|
return null;
|
|
@@ -529,84 +890,48 @@ export async function resolveNodeSourceRuntimeBundle(options = {}) {
|
|
|
529
890
|
return null;
|
|
530
891
|
}
|
|
531
892
|
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
rawConfig: parsed.config,
|
|
548
|
-
tensors: parsed.tensors,
|
|
549
|
-
converterConfig,
|
|
550
|
-
sourceQuantization: parsed.sourceQuantization,
|
|
551
|
-
modelKind: 'transformer',
|
|
552
|
-
architectureHint: parsed.architectureHint,
|
|
553
|
-
architectureConfig: parsed.architecture,
|
|
554
|
-
});
|
|
555
|
-
|
|
556
|
-
const modelId = resolveModelIdHint(
|
|
557
|
-
options.modelId || null,
|
|
558
|
-
plan,
|
|
559
|
-
parsed.sourceKind,
|
|
560
|
-
parsed.sourcePathForModelId
|
|
561
|
-
);
|
|
562
|
-
const hashAlgorithm = converterConfig.manifest.hashAlgorithm;
|
|
563
|
-
const sourceFiles = await addHashesToFileEntries(parsed.sourceFiles, hashAlgorithm);
|
|
564
|
-
const auxiliaryFiles = await addHashesToFileEntries(parsed.auxiliaryFiles, hashAlgorithm);
|
|
565
|
-
const { manifest, shardSources } = await buildSourceRuntimeBundle({
|
|
566
|
-
modelId,
|
|
567
|
-
modelName: modelId,
|
|
568
|
-
modelType: plan.modelType,
|
|
569
|
-
sourceKind: parsed.sourceKind,
|
|
570
|
-
architecture: parsed.architecture,
|
|
571
|
-
architectureHint: parsed.architectureHint,
|
|
572
|
-
rawConfig: parsed.config,
|
|
573
|
-
inference: plan.manifestInference,
|
|
574
|
-
tensors: parsed.tensors,
|
|
575
|
-
embeddingPostprocessor: parsed.embeddingPostprocessor ?? null,
|
|
576
|
-
sourceFiles,
|
|
577
|
-
auxiliaryFiles,
|
|
578
|
-
sourceQuantization: parsed.sourceQuantization,
|
|
579
|
-
quantizationInfo: plan.quantizationInfo,
|
|
580
|
-
hashAlgorithm,
|
|
581
|
-
tokenizerJson: parsed.tokenizerJson,
|
|
582
|
-
tokenizerConfig: parsed.tokenizerConfig,
|
|
583
|
-
tokenizerModelName: parsed.tokenizerModelName,
|
|
584
|
-
tokenizerJsonPath: parsed.tokenizerJsonPath,
|
|
585
|
-
tokenizerConfigPath: parsed.tokenizerConfigPath,
|
|
586
|
-
tokenizerModelPath: parsed.tokenizerModelPath,
|
|
893
|
+
const loadingConfig = resolveLoadingConfig(options.runtimeConfig ?? null);
|
|
894
|
+
const resolvedMemoryBudgetBytes = resolveResidentBudgetBytes(loadingConfig);
|
|
895
|
+
assertSourceRuntimeFitsResidentBudget(parsed, loadingConfig, resolvedMemoryBudgetBytes);
|
|
896
|
+
const {
|
|
897
|
+
model,
|
|
898
|
+
shardSources,
|
|
899
|
+
sourceKind,
|
|
900
|
+
} = await resolveSourceRuntimeBundleFromParsedArtifact({
|
|
901
|
+
parsedArtifact: parsed,
|
|
902
|
+
requestedModelId: options.modelId || null,
|
|
903
|
+
runtimeLabel: 'node source runtime',
|
|
904
|
+
logCategory: 'NodeSourceRuntime',
|
|
905
|
+
hashFileEntries: typeof parsed?.hashFileEntries === 'function'
|
|
906
|
+
? (entries, hashAlgorithm) => parsed.hashFileEntries(entries, hashAlgorithm)
|
|
907
|
+
: addHashesToFileEntries,
|
|
587
908
|
});
|
|
588
909
|
|
|
589
|
-
const readers = buildNodeFileReaders();
|
|
910
|
+
const readers = parsed?.storageReaders ?? buildNodeFileReaders();
|
|
590
911
|
const storageContext = createSourceStorageContext({
|
|
591
|
-
|
|
912
|
+
model,
|
|
592
913
|
shardSources,
|
|
593
914
|
readRange: readers.readRange,
|
|
915
|
+
streamRange: readers.streamRange,
|
|
594
916
|
readText: readers.readText,
|
|
595
917
|
readBinary: readers.readBinary,
|
|
596
918
|
tokenizerJsonPath: parsed.tokenizerJsonPath,
|
|
597
919
|
tokenizerModelPath: parsed.tokenizerModelPath,
|
|
598
920
|
verifyHashes,
|
|
921
|
+
sourceHashesTrusted: true,
|
|
599
922
|
});
|
|
600
923
|
|
|
601
924
|
log.info(
|
|
602
925
|
'NodeSourceRuntime',
|
|
603
|
-
`Source runtime ready: ${
|
|
926
|
+
`Source runtime ready: ${model.modelId} (${sourceKind}, ${parsed.tensors.length} tensors)`
|
|
604
927
|
);
|
|
605
928
|
|
|
606
929
|
return {
|
|
607
|
-
|
|
930
|
+
model,
|
|
931
|
+
manifest: model,
|
|
608
932
|
storageContext,
|
|
609
|
-
sourceKind
|
|
933
|
+
sourceKind,
|
|
610
934
|
sourceRoot: parsed.sourceRoot,
|
|
935
|
+
resolvedMemoryBudgetBytes,
|
|
611
936
|
};
|
|
612
937
|
}
|