doppler-gpu 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -3,19 +3,34 @@
|
|
|
3
3
|
import { initDevice, getDevice, getKernelCapabilities } from '../gpu/device.js';
|
|
4
4
|
import { parseManifest, getExpectedShardHash } from '../formats/rdrr/index.js';
|
|
5
5
|
import { createPipeline } from './pipelines/text.js';
|
|
6
|
+
import { createNodeFileShardStorageContext } from './pipelines/text/init.js';
|
|
6
7
|
import { log as debugLog } from '../debug/index.js';
|
|
7
8
|
import { getRuntimeConfig, setRuntimeConfig } from '../config/runtime.js';
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
9
|
+
|
|
10
|
+
let distributionModulePromise = null;
|
|
11
|
+
let hotSwapManifestModulePromise = null;
|
|
12
|
+
let hotSwapRuntimeModulePromise = null;
|
|
13
|
+
let intentBundleModulePromise = null;
|
|
14
|
+
|
|
15
|
+
async function loadDistributionModule() {
|
|
16
|
+
distributionModulePromise ??= import('../experimental/distribution/shard-delivery.js');
|
|
17
|
+
return distributionModulePromise;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
async function loadHotSwapManifestModule() {
|
|
21
|
+
hotSwapManifestModulePromise ??= import('../experimental/hotswap/manifest.js');
|
|
22
|
+
return hotSwapManifestModulePromise;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
async function loadHotSwapRuntimeModule() {
|
|
26
|
+
hotSwapRuntimeModulePromise ??= import('../experimental/hotswap/runtime.js');
|
|
27
|
+
return hotSwapRuntimeModulePromise;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async function loadIntentBundleModule() {
|
|
31
|
+
intentBundleModulePromise ??= import('../experimental/hotswap/intent-bundle.js');
|
|
32
|
+
return intentBundleModulePromise;
|
|
33
|
+
}
|
|
19
34
|
|
|
20
35
|
|
|
21
36
|
|
|
@@ -166,6 +181,7 @@ export function createHttpShardLoader(baseUrl, manifest, log) {
|
|
|
166
181
|
// Start new load and track it as pending
|
|
167
182
|
const loadPromise = (async () => {
|
|
168
183
|
try {
|
|
184
|
+
const { downloadShard: downloadShardFromDistribution } = await loadDistributionModule();
|
|
169
185
|
const result = await downloadShardFromDistribution(baseUrl, idx, shard, {
|
|
170
186
|
distributionConfig,
|
|
171
187
|
algorithm,
|
|
@@ -201,6 +217,16 @@ export function createHttpShardLoader(baseUrl, manifest, log) {
|
|
|
201
217
|
};
|
|
202
218
|
}
|
|
203
219
|
|
|
220
|
+
export function createHarnessShardStorageContext(modelUrl, manifest, log) {
|
|
221
|
+
const nodeFileStorageContext = createNodeFileShardStorageContext(modelUrl, manifest);
|
|
222
|
+
if (nodeFileStorageContext) {
|
|
223
|
+
return nodeFileStorageContext;
|
|
224
|
+
}
|
|
225
|
+
return {
|
|
226
|
+
loadShard: createHttpShardLoader(modelUrl, manifest, log),
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
|
|
204
230
|
// ============================================================================
|
|
205
231
|
// Pipeline Initialization
|
|
206
232
|
// ============================================================================
|
|
@@ -231,7 +257,14 @@ export async function initializeInference(modelUrl, options = {}) {
|
|
|
231
257
|
const hotSwapConfig = getRuntimeConfig().shared.hotSwap;
|
|
232
258
|
const intentBundleConfig = getRuntimeConfig().shared.intentBundle;
|
|
233
259
|
if (hotSwapConfig.enabled && hotSwapConfig.manifestUrl) {
|
|
234
|
-
const
|
|
260
|
+
const [
|
|
261
|
+
hotSwapManifestModule,
|
|
262
|
+
hotSwapRuntimeModule,
|
|
263
|
+
] = await Promise.all([
|
|
264
|
+
loadHotSwapManifestModule(),
|
|
265
|
+
loadHotSwapRuntimeModule(),
|
|
266
|
+
]);
|
|
267
|
+
const rolloutDecision = hotSwapRuntimeModule.evaluateHotSwapRollout(hotSwapConfig, {
|
|
235
268
|
modelUrl,
|
|
236
269
|
subjectId: options.modelId || null,
|
|
237
270
|
sessionId: options.sessionId || null,
|
|
@@ -242,8 +275,8 @@ export async function initializeInference(modelUrl, options = {}) {
|
|
|
242
275
|
} else {
|
|
243
276
|
onProgress('hotswap', 0.05, 'Loading hot-swap manifest...');
|
|
244
277
|
log(`Hot-swap: loading manifest ${hotSwapConfig.manifestUrl}`);
|
|
245
|
-
const hotSwapManifest = await fetchHotSwapManifest(hotSwapConfig.manifestUrl);
|
|
246
|
-
const verification = await verifyHotSwapManifest(hotSwapManifest, hotSwapConfig, {
|
|
278
|
+
const hotSwapManifest = await hotSwapManifestModule.fetchHotSwapManifest(hotSwapConfig.manifestUrl);
|
|
279
|
+
const verification = await hotSwapManifestModule.verifyHotSwapManifest(hotSwapManifest, hotSwapConfig, {
|
|
247
280
|
source: {
|
|
248
281
|
kind: 'remote',
|
|
249
282
|
isLocal: false,
|
|
@@ -253,7 +286,7 @@ export async function initializeInference(modelUrl, options = {}) {
|
|
|
253
286
|
if (!verification.ok) {
|
|
254
287
|
throw new Error(`Hot-swap manifest rejected: ${verification.reason}`);
|
|
255
288
|
}
|
|
256
|
-
setHotSwapManifest(hotSwapManifest);
|
|
289
|
+
hotSwapRuntimeModule.setHotSwapManifest(hotSwapManifest);
|
|
257
290
|
log(
|
|
258
291
|
`Hot-swap manifest accepted: ${hotSwapManifest.bundleId} (${verification.reason}, rollout=${rolloutDecision.reason})`
|
|
259
292
|
);
|
|
@@ -278,13 +311,14 @@ export async function initializeInference(modelUrl, options = {}) {
|
|
|
278
311
|
const manifest = await fetchManifest(manifestUrl);
|
|
279
312
|
|
|
280
313
|
if (intentBundleConfig.enabled && intentBundleConfig.bundleUrl) {
|
|
314
|
+
const intentBundleModule = await loadIntentBundleModule();
|
|
281
315
|
onProgress('intent', 0.12, 'Loading intent bundle...');
|
|
282
316
|
log(`Intent bundle: loading ${intentBundleConfig.bundleUrl}`);
|
|
283
|
-
const bundle = await fetchIntentBundle(intentBundleConfig.bundleUrl);
|
|
317
|
+
const bundle = await intentBundleModule.fetchIntentBundle(intentBundleConfig.bundleUrl);
|
|
284
318
|
const kernelRegistryVersion = intentBundleConfig.requireKernelRegistryVersion
|
|
285
|
-
? await getKernelRegistryVersion()
|
|
319
|
+
? await intentBundleModule.getKernelRegistryVersion()
|
|
286
320
|
: null;
|
|
287
|
-
const verification = await verifyIntentBundle(bundle, {
|
|
321
|
+
const verification = await intentBundleModule.verifyIntentBundle(bundle, {
|
|
288
322
|
manifest: intentBundleConfig.requireBaseModelHash ? manifest : null,
|
|
289
323
|
kernelRegistryVersion,
|
|
290
324
|
enforceDeterministicOutput: intentBundleConfig.enforceDeterministicOutput,
|
|
@@ -305,7 +339,7 @@ export async function initializeInference(modelUrl, options = {}) {
|
|
|
305
339
|
log(`Model: ${modelLabel}`);
|
|
306
340
|
|
|
307
341
|
// 3. Create shard loader
|
|
308
|
-
const
|
|
342
|
+
const storageContext = createHarnessShardStorageContext(modelUrl, manifest, log);
|
|
309
343
|
|
|
310
344
|
// 4. Build runtime options
|
|
311
345
|
|
|
@@ -318,7 +352,7 @@ export async function initializeInference(modelUrl, options = {}) {
|
|
|
318
352
|
log('Creating pipeline...');
|
|
319
353
|
|
|
320
354
|
const pipeline = await createPipeline( ( (manifest)), {
|
|
321
|
-
storage:
|
|
355
|
+
storage: storageContext,
|
|
322
356
|
gpu: { device },
|
|
323
357
|
runtime,
|
|
324
358
|
baseUrl: modelUrl,
|
|
@@ -67,6 +67,11 @@ export declare class Tokenizer {
|
|
|
67
67
|
* Get vocabulary size
|
|
68
68
|
*/
|
|
69
69
|
getVocabSize(): number;
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Get high-priority token IDs for bounded runtime hot caches, or null when unavailable
|
|
73
|
+
*/
|
|
74
|
+
getHotTokenIds(limit: number): number[] | null;
|
|
70
75
|
}
|
|
71
76
|
|
|
72
77
|
export default Tokenizer;
|
|
@@ -6,6 +6,8 @@ import { TransformersTokenizer, BundledTokenizer } from './tokenizers/bundled.js
|
|
|
6
6
|
import { SentencePieceTokenizer } from './tokenizers/sentencepiece.js';
|
|
7
7
|
import { BPETokenizer } from './tokenizers/bpe.js';
|
|
8
8
|
|
|
9
|
+
const BUNDLED_TOKENIZER_CACHE = new Map();
|
|
10
|
+
|
|
9
11
|
function hasScheme(value) {
|
|
10
12
|
return /^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(value);
|
|
11
13
|
}
|
|
@@ -33,6 +35,28 @@ function parseTokenizerJsonPayload(value) {
|
|
|
33
35
|
throw new Error('Tokenizer JSON loader must return an object, JSON string, or null.');
|
|
34
36
|
}
|
|
35
37
|
|
|
38
|
+
function buildBundledTokenizerCacheKey(modelId, tokenizerConfig) {
|
|
39
|
+
const tokenizerType = typeof tokenizerConfig?.type === 'string'
|
|
40
|
+
? tokenizerConfig.type.trim().toLowerCase()
|
|
41
|
+
: '';
|
|
42
|
+
const tokenizerFile = typeof tokenizerConfig?.file === 'string'
|
|
43
|
+
? tokenizerConfig.file.trim()
|
|
44
|
+
: '';
|
|
45
|
+
if (!tokenizerType || !tokenizerFile) {
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
return JSON.stringify({
|
|
49
|
+
modelId: String(modelId || '').trim() || 'unknown',
|
|
50
|
+
tokenizerType,
|
|
51
|
+
tokenizerFile,
|
|
52
|
+
vocabSize: Number.isFinite(tokenizerConfig?.vocabSize) ? tokenizerConfig.vocabSize : null,
|
|
53
|
+
eosToken: tokenizerConfig?.eosToken ?? null,
|
|
54
|
+
bosToken: tokenizerConfig?.bosToken ?? null,
|
|
55
|
+
padToken: tokenizerConfig?.padToken ?? null,
|
|
56
|
+
unkToken: tokenizerConfig?.unkToken ?? null,
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
|
|
36
60
|
export class Tokenizer {
|
|
37
61
|
|
|
38
62
|
backend = null;
|
|
@@ -71,6 +95,15 @@ export class Tokenizer {
|
|
|
71
95
|
if (isBundled && tokenizerConfig.file) {
|
|
72
96
|
tokenizerConfig.deferSpecialTokens = true;
|
|
73
97
|
log.info('Tokenizer', `Loading ${tokenizerConfig.type} tokenizer from ${tokenizerConfig.file}`);
|
|
98
|
+
const cacheKey = buildBundledTokenizerCacheKey(modelId, tokenizerConfig);
|
|
99
|
+
const cachedBackend = cacheKey ? BUNDLED_TOKENIZER_CACHE.get(cacheKey) : null;
|
|
100
|
+
if (cachedBackend) {
|
|
101
|
+
log.info('Tokenizer', `Bundled tokenizer cache hit: ${tokenizerConfig.file}`);
|
|
102
|
+
this.backend = cachedBackend;
|
|
103
|
+
this.config = tokenizerConfig;
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
|
|
74
107
|
this.backend = new BundledTokenizer(tokenizerConfig);
|
|
75
108
|
|
|
76
109
|
const baseUrl = options.baseUrl;
|
|
@@ -121,6 +154,9 @@ export class Tokenizer {
|
|
|
121
154
|
|
|
122
155
|
if (tokenizerJson) {
|
|
123
156
|
(this.backend).load(tokenizerJson);
|
|
157
|
+
if (cacheKey) {
|
|
158
|
+
BUNDLED_TOKENIZER_CACHE.set(cacheKey, this.backend);
|
|
159
|
+
}
|
|
124
160
|
this.config = tokenizerConfig;
|
|
125
161
|
return;
|
|
126
162
|
}
|
|
@@ -246,6 +282,13 @@ export class Tokenizer {
|
|
|
246
282
|
getVocabSize() {
|
|
247
283
|
return this.backend?.getVocabSize() || 0;
|
|
248
284
|
}
|
|
285
|
+
|
|
286
|
+
getHotTokenIds(limit) {
|
|
287
|
+
if (!this.backend || typeof this.backend.getHotTokenIds !== 'function') {
|
|
288
|
+
return null;
|
|
289
|
+
}
|
|
290
|
+
return this.backend.getHotTokenIds(limit);
|
|
291
|
+
}
|
|
249
292
|
}
|
|
250
293
|
|
|
251
294
|
export default Tokenizer;
|
|
@@ -32,6 +32,11 @@ export declare abstract class BaseTokenizer implements TokenizerBackend {
|
|
|
32
32
|
*/
|
|
33
33
|
getVocabSize(): number;
|
|
34
34
|
|
|
35
|
+
/**
|
|
36
|
+
* Get high-priority token IDs for bounded runtime hot caches, or null when unavailable
|
|
37
|
+
*/
|
|
38
|
+
getHotTokenIds(limit: number): number[] | null;
|
|
39
|
+
|
|
35
40
|
/**
|
|
36
41
|
* Check if token is special
|
|
37
42
|
*/
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export interface BundledTokenizerBehaviorFlags {
|
|
2
|
+
addBosToken: boolean | null;
|
|
3
|
+
addEosToken: boolean | null;
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
export function inferBundledTokenizerBehaviorFlags(
|
|
7
|
+
tokenizerJson: unknown,
|
|
8
|
+
specialTokens?: { bos?: number | null, eos?: number | null } | null
|
|
9
|
+
): BundledTokenizerBehaviorFlags;
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
function normalizeSpecialTokenLabel(value) {
|
|
2
|
+
if (typeof value !== 'string') {
|
|
3
|
+
return '';
|
|
4
|
+
}
|
|
5
|
+
return value.trim().toLowerCase();
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
function isBosLikeLabel(value) {
|
|
9
|
+
const normalized = normalizeSpecialTokenLabel(value);
|
|
10
|
+
return normalized === '<bos>' || normalized === '<s>' || normalized.includes('bos');
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function isEosLikeLabel(value) {
|
|
14
|
+
const normalized = normalizeSpecialTokenLabel(value);
|
|
15
|
+
return normalized === '<eos>' || normalized === '</s>' || normalized.includes('eos');
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function classifySpecialTokenReference(ref, specialTokens) {
|
|
19
|
+
if (typeof ref === 'number' && Number.isFinite(ref)) {
|
|
20
|
+
if (specialTokens?.bos === ref) return 'bos';
|
|
21
|
+
if (specialTokens?.eos === ref) return 'eos';
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
if (Array.isArray(ref)) {
|
|
26
|
+
for (const item of ref) {
|
|
27
|
+
const kind = classifySpecialTokenReference(item, specialTokens);
|
|
28
|
+
if (kind) {
|
|
29
|
+
return kind;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if (ref && typeof ref === 'object') {
|
|
36
|
+
if (typeof ref.id === 'number' && Number.isFinite(ref.id)) {
|
|
37
|
+
return classifySpecialTokenReference(ref.id, specialTokens);
|
|
38
|
+
}
|
|
39
|
+
if (Array.isArray(ref.ids)) {
|
|
40
|
+
return classifySpecialTokenReference(ref.ids, specialTokens);
|
|
41
|
+
}
|
|
42
|
+
if (Array.isArray(ref.tokens)) {
|
|
43
|
+
return classifySpecialTokenReference(ref.tokens, specialTokens);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (typeof ref === 'string') {
|
|
48
|
+
if (isBosLikeLabel(ref)) return 'bos';
|
|
49
|
+
if (isEosLikeLabel(ref)) return 'eos';
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function getTemplateBoundaryKind(template, boundary, specialTokens) {
|
|
56
|
+
if (!Array.isArray(template) || template.length === 0) {
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const entries = boundary === 'last'
|
|
61
|
+
? [...template].reverse()
|
|
62
|
+
: template;
|
|
63
|
+
|
|
64
|
+
for (const entry of entries) {
|
|
65
|
+
const special = entry?.SpecialToken;
|
|
66
|
+
if (!special) {
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const kind = classifySpecialTokenReference(
|
|
71
|
+
special.tokens ?? special.ids ?? special.id ?? null,
|
|
72
|
+
specialTokens
|
|
73
|
+
);
|
|
74
|
+
if (kind) {
|
|
75
|
+
return kind;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function inferBundledTokenizerBehaviorFlags(tokenizerJson, specialTokens = null) {
|
|
83
|
+
if (!tokenizerJson || typeof tokenizerJson !== 'object' || Array.isArray(tokenizerJson)) {
|
|
84
|
+
return { addBosToken: null, addEosToken: null };
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const postProcessor = tokenizerJson.post_processor;
|
|
88
|
+
if (!postProcessor || postProcessor.type !== 'TemplateProcessing') {
|
|
89
|
+
return { addBosToken: null, addEosToken: null };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const single = Array.isArray(postProcessor.single) ? postProcessor.single : null;
|
|
93
|
+
const firstKind = getTemplateBoundaryKind(single, 'first', specialTokens);
|
|
94
|
+
const lastKind = getTemplateBoundaryKind(single, 'last', specialTokens);
|
|
95
|
+
|
|
96
|
+
return {
|
|
97
|
+
addBosToken: firstKind === 'bos' ? true : null,
|
|
98
|
+
addEosToken: lastKind === 'eos' ? true : null,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
@@ -39,6 +39,8 @@ export declare class TransformersTokenizer extends BaseTokenizer {
|
|
|
39
39
|
* Batch encode multiple texts
|
|
40
40
|
*/
|
|
41
41
|
batchEncode(texts: string[]): number[][];
|
|
42
|
+
|
|
43
|
+
getHotTokenIds(limit: number): number[] | null;
|
|
42
44
|
}
|
|
43
45
|
|
|
44
46
|
/**
|
|
@@ -60,4 +62,6 @@ export declare class BundledTokenizer extends BaseTokenizer {
|
|
|
60
62
|
encode(text: string): number[];
|
|
61
63
|
|
|
62
64
|
decode(ids: number[], skipSpecialTokens?: boolean, trim?: boolean): string;
|
|
65
|
+
|
|
66
|
+
getHotTokenIds(limit: number): number[] | null;
|
|
63
67
|
}
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import { BaseTokenizer } from './base.js';
|
|
4
4
|
import { log } from '../../debug/index.js';
|
|
5
5
|
import { getRuntimeConfig } from '../../config/runtime.js';
|
|
6
|
+
import { inferBundledTokenizerBehaviorFlags } from './behavior-flags.js';
|
|
6
7
|
|
|
7
8
|
function pickCandidate(...values) {
|
|
8
9
|
for (const value of values) {
|
|
@@ -94,6 +95,78 @@ function resolveByteLevelPretokenizerConfig(preTokenizer) {
|
|
|
94
95
|
};
|
|
95
96
|
}
|
|
96
97
|
|
|
98
|
+
function isSpecialLikeHotTokenCandidate(token) {
|
|
99
|
+
if (typeof token !== 'string' || token.length === 0) {
|
|
100
|
+
return true;
|
|
101
|
+
}
|
|
102
|
+
if ((token.startsWith('<') && token.endsWith('>')) || (token.startsWith('[') && token.endsWith(']'))) {
|
|
103
|
+
return true;
|
|
104
|
+
}
|
|
105
|
+
return /unused|reserved|multimodal|image|video|audio/i.test(token);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function normalizeBpeHotTokenCandidate(token) {
|
|
109
|
+
if (typeof token !== 'string') {
|
|
110
|
+
return '';
|
|
111
|
+
}
|
|
112
|
+
return token
|
|
113
|
+
.replace(/^[▁Ġ]+/, '')
|
|
114
|
+
.replace(/Ċ/g, '\n');
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function scoreBpeHotTokenCandidate(token, id) {
|
|
118
|
+
if (isSpecialLikeHotTokenCandidate(token)) {
|
|
119
|
+
return Number.NEGATIVE_INFINITY;
|
|
120
|
+
}
|
|
121
|
+
const normalized = normalizeBpeHotTokenCandidate(token);
|
|
122
|
+
if (normalized.length === 0) {
|
|
123
|
+
return Number.NEGATIVE_INFINITY;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const hasBoundaryMarker = token.startsWith('▁') || token.startsWith('Ġ');
|
|
127
|
+
const isAscii = /^[\x00-\x7F]+$/.test(normalized);
|
|
128
|
+
const isLowerAlpha = /^[a-z]+$/.test(normalized);
|
|
129
|
+
const isTitleAlpha = /^[A-Z][a-z]+$/.test(normalized);
|
|
130
|
+
const isAlpha = /^[A-Za-z]+$/.test(normalized);
|
|
131
|
+
const isPunctuation = /^[.,!?;:'"()%-]+$/.test(normalized);
|
|
132
|
+
const isDigits = /^\d+$/.test(normalized);
|
|
133
|
+
const length = normalized.length;
|
|
134
|
+
|
|
135
|
+
let score = 0;
|
|
136
|
+
if (hasBoundaryMarker) score += 40;
|
|
137
|
+
if (isLowerAlpha) score += 32;
|
|
138
|
+
else if (isTitleAlpha) score += 24;
|
|
139
|
+
else if (isAlpha) score += 20;
|
|
140
|
+
if (isPunctuation) score += 20;
|
|
141
|
+
if (isDigits) score += 8;
|
|
142
|
+
if (isAscii) score += 12;
|
|
143
|
+
|
|
144
|
+
if (length === 1) score += hasBoundaryMarker ? 8 : 2;
|
|
145
|
+
else if (length <= 4) score += 18 - (length * 2);
|
|
146
|
+
else if (length <= 8) score += 12 - (length - 4);
|
|
147
|
+
else if (length <= 12) score += 4 - (length - 8);
|
|
148
|
+
else score -= Math.min(12, length - 12);
|
|
149
|
+
|
|
150
|
+
score -= id / 1e7;
|
|
151
|
+
return score;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function rankFallbackBpeHotTokenIds(reverseVocab, limit, isSpecialToken) {
|
|
155
|
+
const ranked = [];
|
|
156
|
+
for (const [id, token] of reverseVocab.entries()) {
|
|
157
|
+
if (typeof isSpecialToken === 'function' && isSpecialToken(id)) {
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
const score = scoreBpeHotTokenCandidate(token, id);
|
|
161
|
+
if (!Number.isFinite(score)) {
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
ranked.push({ id, score });
|
|
165
|
+
}
|
|
166
|
+
ranked.sort((a, b) => b.score - a.score || a.id - b.id);
|
|
167
|
+
return ranked.slice(0, limit).map((entry) => entry.id);
|
|
168
|
+
}
|
|
169
|
+
|
|
97
170
|
function registerAddedTokens(addedTokens, vocab, reverseVocab, patterns, specialTokenIds, derivedSpecialTokens = null) {
|
|
98
171
|
let maxId = -1;
|
|
99
172
|
for (const token of addedTokens) {
|
|
@@ -187,6 +260,11 @@ export class TransformersTokenizer extends BaseTokenizer {
|
|
|
187
260
|
batchEncode(texts) {
|
|
188
261
|
return texts.map(t => this.encode(t));
|
|
189
262
|
}
|
|
263
|
+
|
|
264
|
+
getHotTokenIds(limit) {
|
|
265
|
+
void limit;
|
|
266
|
+
return null;
|
|
267
|
+
}
|
|
190
268
|
}
|
|
191
269
|
|
|
192
270
|
|
|
@@ -437,15 +515,18 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
437
515
|
const byteLevelPretokenizer = resolveByteLevelPretokenizerConfig(hf.pre_tokenizer);
|
|
438
516
|
const configuredAddBosToken = this.addBosToken;
|
|
439
517
|
const configuredAddEosToken = this.addEosToken;
|
|
518
|
+
const inferredFlags = inferBundledTokenizerBehaviorFlags(hf, this.specialTokens);
|
|
440
519
|
this.addBosToken =
|
|
441
520
|
hf.add_bos_token
|
|
442
521
|
?? hf.addBosToken
|
|
443
522
|
?? configuredAddBosToken
|
|
523
|
+
?? inferredFlags.addBosToken
|
|
444
524
|
?? runtimeDefaults.addBosToken;
|
|
445
525
|
this.addEosToken =
|
|
446
526
|
hf.add_eos_token
|
|
447
527
|
?? hf.addEosToken
|
|
448
528
|
?? configuredAddEosToken
|
|
529
|
+
?? inferredFlags.addEosToken
|
|
449
530
|
?? runtimeDefaults.addEosToken;
|
|
450
531
|
if (this.addBosToken && this.specialTokens.bos == null) {
|
|
451
532
|
throw new Error('[Tokenizer] addBosToken is enabled but bos token is missing.');
|
|
@@ -621,15 +702,18 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
621
702
|
const byteLevelPretokenizer = resolveByteLevelPretokenizerConfig(tokenizerJson.pre_tokenizer);
|
|
622
703
|
const configuredAddBosToken = this.addBosToken;
|
|
623
704
|
const configuredAddEosToken = this.addEosToken;
|
|
705
|
+
const inferredFlags = inferBundledTokenizerBehaviorFlags(tokenizerJson, this.specialTokens);
|
|
624
706
|
this.addBosToken =
|
|
625
707
|
tokenizerJson.addBosToken
|
|
626
708
|
?? tokenizerJson.add_bos_token
|
|
627
709
|
?? configuredAddBosToken
|
|
710
|
+
?? inferredFlags.addBosToken
|
|
628
711
|
?? runtimeDefaults.addBosToken;
|
|
629
712
|
this.addEosToken =
|
|
630
713
|
tokenizerJson.addEosToken
|
|
631
714
|
?? tokenizerJson.add_eos_token
|
|
632
715
|
?? configuredAddEosToken
|
|
716
|
+
?? inferredFlags.addEosToken
|
|
633
717
|
?? runtimeDefaults.addEosToken;
|
|
634
718
|
if (this.addBosToken && this.specialTokens.bos == null) {
|
|
635
719
|
throw new Error('[Tokenizer] addBosToken is enabled but bos token is missing.');
|
|
@@ -1006,4 +1090,30 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
1006
1090
|
// Only trim when requested (not during streaming where spaces matter)
|
|
1007
1091
|
return trim ? result.trim() : result;
|
|
1008
1092
|
}
|
|
1093
|
+
|
|
1094
|
+
getHotTokenIds(limit) {
|
|
1095
|
+
const resolvedLimit = Math.trunc(Number(limit));
|
|
1096
|
+
if (!Number.isFinite(resolvedLimit) || resolvedLimit <= 0) {
|
|
1097
|
+
return [];
|
|
1098
|
+
}
|
|
1099
|
+
if (this.#type === 'bpe' && (!Array.isArray(this.#scores) || this.#scores.length === 0)) {
|
|
1100
|
+
return rankFallbackBpeHotTokenIds(
|
|
1101
|
+
this.#reverseVocab,
|
|
1102
|
+
resolvedLimit,
|
|
1103
|
+
(tokenId) => this.isSpecialToken(tokenId)
|
|
1104
|
+
);
|
|
1105
|
+
}
|
|
1106
|
+
if (!Array.isArray(this.#scores) || this.#scores.length === 0) {
|
|
1107
|
+
return null;
|
|
1108
|
+
}
|
|
1109
|
+
const ranked = [];
|
|
1110
|
+
for (let id = 0; id < this.#scores.length; id += 1) {
|
|
1111
|
+
if (this.isSpecialToken(id)) {
|
|
1112
|
+
continue;
|
|
1113
|
+
}
|
|
1114
|
+
ranked.push({ id, score: Number(this.#scores[id] ?? 0) });
|
|
1115
|
+
}
|
|
1116
|
+
ranked.sort((a, b) => b.score - a.score || a.id - b.id);
|
|
1117
|
+
return ranked.slice(0, resolvedLimit).map((entry) => entry.id);
|
|
1118
|
+
}
|
|
1009
1119
|
}
|
|
@@ -58,6 +58,22 @@ export class SentencePieceTokenizer extends BaseTokenizer {
|
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
+
getHotTokenIds(limit) {
|
|
62
|
+
const resolvedLimit = Math.trunc(Number(limit));
|
|
63
|
+
if (!Number.isFinite(resolvedLimit) || resolvedLimit <= 0) {
|
|
64
|
+
return [];
|
|
65
|
+
}
|
|
66
|
+
const ranked = [];
|
|
67
|
+
for (const piece of this.#pieces.values()) {
|
|
68
|
+
if (!piece || this.isSpecialToken(piece.id)) {
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
ranked.push({ id: piece.id, score: Number(piece.score ?? 0) });
|
|
72
|
+
}
|
|
73
|
+
ranked.sort((a, b) => b.score - a.score || a.id - b.id);
|
|
74
|
+
return ranked.slice(0, resolvedLimit).map((entry) => entry.id);
|
|
75
|
+
}
|
|
76
|
+
|
|
61
77
|
|
|
62
78
|
async #parseModelProto(buffer) {
|
|
63
79
|
const view = new DataView(buffer);
|
|
@@ -67,6 +67,8 @@ export interface TokenizerBackend {
|
|
|
67
67
|
decode(tokens: number[], skipSpecial?: boolean, trim?: boolean): string;
|
|
68
68
|
/** Get vocabulary size */
|
|
69
69
|
getVocabSize(): number;
|
|
70
|
+
/** Get high-priority token IDs for bounded runtime hot caches, or null when unavailable */
|
|
71
|
+
getHotTokenIds?(limit: number): number[] | null;
|
|
70
72
|
/** Get special token IDs */
|
|
71
73
|
specialTokens: SpecialTokens;
|
|
72
74
|
}
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
import type { MemoryCapabilities } from '../memory/capability.js';
|
|
14
14
|
import type { HeapManager } from '../memory/heap-manager.js';
|
|
15
15
|
import type { RDRRManifest } from '../formats/rdrr/index.js';
|
|
16
|
+
import type { RuntimeModelContract } from '../inference/runtime-model.js';
|
|
16
17
|
import type { WeightBuffer, WeightLayout, CpuWeightBuffer } from '../gpu/weight-buffer.js';
|
|
17
18
|
import type { ExpertCache, CacheStats } from './experts/expert-cache.js';
|
|
18
19
|
import type { ExpertWeights } from './weights.js';
|
|
@@ -21,6 +22,7 @@ import type { LoadedEmbeddingPostprocessor } from './final-weights-loader.js';
|
|
|
21
22
|
import type {
|
|
22
23
|
TensorLocation,
|
|
23
24
|
LayerWeights,
|
|
25
|
+
PerLayerInputWeights,
|
|
24
26
|
LoadProgress,
|
|
25
27
|
LoadOptions,
|
|
26
28
|
CustomShardLoader,
|
|
@@ -33,6 +35,7 @@ import type {
|
|
|
33
35
|
import type { ShardCache } from './shard-cache.js';
|
|
34
36
|
import type { LoadingConfigSchema } from '../config/schema/loading.schema.js';
|
|
35
37
|
import type { LoaderDebugConfigSchema } from '../config/schema/debug.schema.js';
|
|
38
|
+
import type { ExecutionV1PerLayerInputsSessionSchema } from '../config/schema/execution-v1.schema.js';
|
|
36
39
|
|
|
37
40
|
// Re-export types for backward compatibility
|
|
38
41
|
export type {
|
|
@@ -59,7 +62,7 @@ export declare class DopplerLoader {
|
|
|
59
62
|
isUnifiedMemory: boolean;
|
|
60
63
|
|
|
61
64
|
// Manifest and model info
|
|
62
|
-
manifest: RDRRManifest | null;
|
|
65
|
+
manifest: (RDRRManifest | RuntimeModelContract) | null;
|
|
63
66
|
modelId: string | null;
|
|
64
67
|
isMoE: boolean;
|
|
65
68
|
|
|
@@ -71,6 +74,7 @@ export declare class DopplerLoader {
|
|
|
71
74
|
lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
|
|
72
75
|
finalNorm: GPUBuffer | Float32Array | null;
|
|
73
76
|
embeddingPostprocessor: LoadedEmbeddingPostprocessor | null;
|
|
77
|
+
perLayerInputWeights: PerLayerInputWeights | null;
|
|
74
78
|
|
|
75
79
|
// Memory management
|
|
76
80
|
heapManager: HeapManager | null;
|
|
@@ -92,6 +96,7 @@ export declare class DopplerLoader {
|
|
|
92
96
|
// Q4K layout: 'row' = fused kernel compatible (fast), 'col' = dequant fallback
|
|
93
97
|
q4kLayout: 'row' | 'col' | null;
|
|
94
98
|
keepF32Weights: boolean;
|
|
99
|
+
q4kMaterializationMode: 'dense' | 'fused' | 'mixed';
|
|
95
100
|
|
|
96
101
|
constructor(loadingConfig?: LoadingConfigSchema);
|
|
97
102
|
|
|
@@ -101,15 +106,17 @@ export declare class DopplerLoader {
|
|
|
101
106
|
|
|
102
107
|
setLoaderDebugConfig(loaderDebug: LoaderDebugConfigSchema | null): void;
|
|
103
108
|
|
|
109
|
+
setPerLayerInputSession(sessionConfig: ExecutionV1PerLayerInputsSessionSchema | null): void;
|
|
110
|
+
|
|
104
111
|
setCustomShardLoader(loadShardFn: CustomShardLoader, options?: CustomShardLoaderOptions): void;
|
|
105
112
|
|
|
106
113
|
setTensorsJsonUrl(url: string | null): void;
|
|
107
114
|
|
|
108
115
|
init(): Promise<void>;
|
|
109
116
|
|
|
110
|
-
setManifest(manifest: RDRRManifest): void;
|
|
117
|
+
setManifest(manifest: RDRRManifest | RuntimeModelContract): void;
|
|
111
118
|
|
|
112
|
-
loadLoRAWeights(manifest: RDRRManifest): Promise<LoRAAdapter>;
|
|
119
|
+
loadLoRAWeights(manifest: RDRRManifest | RuntimeModelContract): Promise<LoRAAdapter>;
|
|
113
120
|
|
|
114
121
|
load(modelId: string, options?: LoadOptions): Promise<ModelConfig>;
|
|
115
122
|
|