doppler-gpu 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -0,0 +1,633 @@
|
|
|
1
|
+
const TEXT_DECODER = new TextDecoder();
|
|
2
|
+
const TFLITE_PAGE_BYTES = 16 * 1024;
|
|
3
|
+
|
|
4
|
+
export const TFLITE_FILE_IDENTIFIER = 'TFL3';
|
|
5
|
+
|
|
6
|
+
export const TFLITE_TENSOR_TYPE = Object.freeze({
|
|
7
|
+
FLOAT32: 0,
|
|
8
|
+
FLOAT16: 1,
|
|
9
|
+
INT32: 2,
|
|
10
|
+
UINT8: 3,
|
|
11
|
+
INT64: 4,
|
|
12
|
+
STRING: 5,
|
|
13
|
+
BOOL: 6,
|
|
14
|
+
INT16: 7,
|
|
15
|
+
COMPLEX64: 8,
|
|
16
|
+
INT8: 9,
|
|
17
|
+
FLOAT64: 10,
|
|
18
|
+
COMPLEX128: 11,
|
|
19
|
+
UINT64: 12,
|
|
20
|
+
RESOURCE: 13,
|
|
21
|
+
VARIANT: 14,
|
|
22
|
+
UINT32: 15,
|
|
23
|
+
UINT16: 16,
|
|
24
|
+
INT4: 17,
|
|
25
|
+
BFLOAT16: 18,
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
export const TFLITE_TENSOR_TYPE_NAME = Object.freeze(Object.fromEntries(
|
|
29
|
+
Object.entries(TFLITE_TENSOR_TYPE).map(([name, value]) => [value, name])
|
|
30
|
+
));
|
|
31
|
+
|
|
32
|
+
export const TFLITE_TENSOR_DTYPE_MAP = Object.freeze({
|
|
33
|
+
[TFLITE_TENSOR_TYPE.FLOAT32]: 'F32',
|
|
34
|
+
[TFLITE_TENSOR_TYPE.FLOAT16]: 'F16',
|
|
35
|
+
[TFLITE_TENSOR_TYPE.BFLOAT16]: 'BF16',
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
export const TFLITE_TENSOR_SOURCE_DTYPE_MAP = Object.freeze({
|
|
39
|
+
[TFLITE_TENSOR_TYPE.FLOAT32]: 'F32',
|
|
40
|
+
[TFLITE_TENSOR_TYPE.FLOAT16]: 'F16',
|
|
41
|
+
[TFLITE_TENSOR_TYPE.BFLOAT16]: 'BF16',
|
|
42
|
+
[TFLITE_TENSOR_TYPE.INT8]: 'INT8',
|
|
43
|
+
[TFLITE_TENSOR_TYPE.UINT8]: 'UINT8',
|
|
44
|
+
[TFLITE_TENSOR_TYPE.INT4]: 'INT4',
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
export const TFLITE_DTYPE_SIZE = Object.freeze({
|
|
48
|
+
F32: 4,
|
|
49
|
+
F16: 2,
|
|
50
|
+
BF16: 2,
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
function normalizeParseOptions(options) {
|
|
54
|
+
return {
|
|
55
|
+
allowPackedQuantization: options?.allowPackedQuantization === true,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function toUint8Array(value, label) {
|
|
60
|
+
if (value instanceof Uint8Array) {
|
|
61
|
+
return value;
|
|
62
|
+
}
|
|
63
|
+
if (value instanceof ArrayBuffer) {
|
|
64
|
+
return new Uint8Array(value);
|
|
65
|
+
}
|
|
66
|
+
throw new Error(`${label} must return ArrayBuffer or Uint8Array.`);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function normalizeSourceLabel(source) {
|
|
70
|
+
const raw = typeof source?.name === 'string' ? source.name.trim() : '';
|
|
71
|
+
return raw || 'TFLite source';
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function decodeAscii(bytes) {
|
|
75
|
+
return Array.from(bytes, (value) => String.fromCharCode(value)).join('');
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function computeTensorElementCount(shape, tensorName) {
|
|
79
|
+
if (!Array.isArray(shape)) {
|
|
80
|
+
throw new Error(`TFLite tensor "${tensorName}" is missing shape.`);
|
|
81
|
+
}
|
|
82
|
+
let total = 1;
|
|
83
|
+
for (let index = 0; index < shape.length; index++) {
|
|
84
|
+
const rawDim = Number(shape[index]);
|
|
85
|
+
if (!Number.isFinite(rawDim) || Math.floor(rawDim) !== rawDim || rawDim < 0) {
|
|
86
|
+
throw new Error(
|
|
87
|
+
`TFLite tensor "${tensorName}" has invalid shape[${index}] (${shape[index]}).`
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
total *= rawDim;
|
|
91
|
+
}
|
|
92
|
+
return total;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function computeTFLiteSourceByteSize(elementCount, sourceDtype, tensorName) {
|
|
96
|
+
if (!Number.isInteger(elementCount) || elementCount < 0) {
|
|
97
|
+
throw new Error(`TFLite tensor "${tensorName}" has invalid element count (${elementCount}).`);
|
|
98
|
+
}
|
|
99
|
+
if (sourceDtype === 'INT4') {
|
|
100
|
+
return Math.ceil(elementCount / 2);
|
|
101
|
+
}
|
|
102
|
+
const byteSize = TFLITE_DTYPE_SIZE[sourceDtype];
|
|
103
|
+
if (Number.isInteger(byteSize) && byteSize >= 0) {
|
|
104
|
+
return elementCount * byteSize;
|
|
105
|
+
}
|
|
106
|
+
if (sourceDtype === 'INT8' || sourceDtype === 'UINT8') {
|
|
107
|
+
return elementCount;
|
|
108
|
+
}
|
|
109
|
+
throw new Error(
|
|
110
|
+
`TFLite tensor "${tensorName}" uses unsupported source dtype ${sourceDtype}.`
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function resolveDirectSourceQuantization(tensors) {
|
|
115
|
+
const dtypes = new Set();
|
|
116
|
+
for (const tensor of tensors) {
|
|
117
|
+
const dtype = String(tensor?.dtype || '').trim().toUpperCase();
|
|
118
|
+
if (dtype) {
|
|
119
|
+
dtypes.add(dtype);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (dtypes.size === 0) {
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
if (dtypes.size === 1) {
|
|
126
|
+
return [...dtypes][0];
|
|
127
|
+
}
|
|
128
|
+
const allSupportedFloat = Array.from(dtypes).every((dtype) => dtype === 'F16' || dtype === 'BF16' || dtype === 'F32');
|
|
129
|
+
return allSupportedFloat ? 'F32' : null;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function createMemoryRangeSource(buffer, label = 'TFLite buffer') {
|
|
133
|
+
const bytes = toUint8Array(buffer, label);
|
|
134
|
+
return {
|
|
135
|
+
name: label,
|
|
136
|
+
size: bytes.byteLength,
|
|
137
|
+
async readRange(offset, length) {
|
|
138
|
+
if (!Number.isFinite(offset) || !Number.isFinite(length) || length <= 0) {
|
|
139
|
+
return new Uint8Array(0);
|
|
140
|
+
}
|
|
141
|
+
const start = Math.max(0, Math.floor(offset));
|
|
142
|
+
const end = Math.min(bytes.byteLength, start + Math.max(0, Math.floor(length)));
|
|
143
|
+
return bytes.slice(start, end);
|
|
144
|
+
},
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function createPagedSourceReader(source) {
|
|
149
|
+
if (!source || typeof source.readRange !== 'function') {
|
|
150
|
+
throw new Error('TFLite parser: source.readRange(offset, length) is required.');
|
|
151
|
+
}
|
|
152
|
+
const size = Number(source.size);
|
|
153
|
+
if (!Number.isFinite(size) || size < 8) {
|
|
154
|
+
throw new Error('TFLite parser: source.size must be a finite byte length >= 8.');
|
|
155
|
+
}
|
|
156
|
+
const label = normalizeSourceLabel(source);
|
|
157
|
+
const pageCache = new Map();
|
|
158
|
+
|
|
159
|
+
async function loadPage(pageIndex) {
|
|
160
|
+
const cached = pageCache.get(pageIndex);
|
|
161
|
+
if (cached) {
|
|
162
|
+
return cached;
|
|
163
|
+
}
|
|
164
|
+
const pageOffset = pageIndex * TFLITE_PAGE_BYTES;
|
|
165
|
+
const pageLength = Math.min(TFLITE_PAGE_BYTES, size - pageOffset);
|
|
166
|
+
const raw = await source.readRange(pageOffset, pageLength);
|
|
167
|
+
const bytes = toUint8Array(raw, `${label} page ${pageIndex}`);
|
|
168
|
+
if (bytes.byteLength !== pageLength) {
|
|
169
|
+
throw new Error(
|
|
170
|
+
`TFLite parser: short read for ${label} page ${pageIndex}. ` +
|
|
171
|
+
`Expected ${pageLength} bytes, got ${bytes.byteLength}.`
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
pageCache.set(pageIndex, bytes);
|
|
175
|
+
return bytes;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
async function readSlice(offset, length) {
|
|
179
|
+
if (!Number.isFinite(offset) || !Number.isFinite(length)) {
|
|
180
|
+
throw new Error(`TFLite parser: invalid read request (${offset}, ${length}).`);
|
|
181
|
+
}
|
|
182
|
+
const start = Math.max(0, Math.floor(offset));
|
|
183
|
+
const byteLength = Math.max(0, Math.floor(length));
|
|
184
|
+
if (start + byteLength > size) {
|
|
185
|
+
throw new Error(
|
|
186
|
+
`TFLite parser: read exceeds ${label} bounds (offset=${start}, length=${byteLength}, size=${size}).`
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
if (byteLength === 0) {
|
|
190
|
+
return new Uint8Array(0);
|
|
191
|
+
}
|
|
192
|
+
const pageStart = Math.floor(start / TFLITE_PAGE_BYTES);
|
|
193
|
+
const pageEnd = Math.floor((start + byteLength - 1) / TFLITE_PAGE_BYTES);
|
|
194
|
+
if (pageStart === pageEnd) {
|
|
195
|
+
const page = await loadPage(pageStart);
|
|
196
|
+
const sliceOffset = start - pageStart * TFLITE_PAGE_BYTES;
|
|
197
|
+
return page.subarray(sliceOffset, sliceOffset + byteLength);
|
|
198
|
+
}
|
|
199
|
+
const out = new Uint8Array(byteLength);
|
|
200
|
+
let writeOffset = 0;
|
|
201
|
+
for (let pageIndex = pageStart; pageIndex <= pageEnd; pageIndex++) {
|
|
202
|
+
const page = await loadPage(pageIndex);
|
|
203
|
+
const pageBase = pageIndex * TFLITE_PAGE_BYTES;
|
|
204
|
+
const sliceStart = pageIndex === pageStart ? start - pageBase : 0;
|
|
205
|
+
const sliceEnd = pageIndex === pageEnd
|
|
206
|
+
? (start + byteLength) - pageBase
|
|
207
|
+
: page.byteLength;
|
|
208
|
+
const slice = page.subarray(sliceStart, sliceEnd);
|
|
209
|
+
out.set(slice, writeOffset);
|
|
210
|
+
writeOffset += slice.byteLength;
|
|
211
|
+
}
|
|
212
|
+
return out;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
async function readInt32(offset) {
|
|
216
|
+
const bytes = await readSlice(offset, 4);
|
|
217
|
+
return new DataView(bytes.buffer, bytes.byteOffset, 4).getInt32(0, true);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async function readUint32(offset) {
|
|
221
|
+
const bytes = await readSlice(offset, 4);
|
|
222
|
+
return new DataView(bytes.buffer, bytes.byteOffset, 4).getUint32(0, true);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
async function readUint16(offset) {
|
|
226
|
+
const bytes = await readSlice(offset, 2);
|
|
227
|
+
return new DataView(bytes.buffer, bytes.byteOffset, 2).getUint16(0, true);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
async function readUint8(offset) {
|
|
231
|
+
const bytes = await readSlice(offset, 1);
|
|
232
|
+
return bytes[0];
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
async function readFloat32(offset) {
|
|
236
|
+
const bytes = await readSlice(offset, 4);
|
|
237
|
+
return new DataView(bytes.buffer, bytes.byteOffset, 4).getFloat32(0, true);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
async function readBigInt64(offset) {
|
|
241
|
+
const bytes = await readSlice(offset, 8);
|
|
242
|
+
return new DataView(bytes.buffer, bytes.byteOffset, 8).getBigInt64(0, true);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
async function readBigUint64(offset) {
|
|
246
|
+
const bytes = await readSlice(offset, 8);
|
|
247
|
+
return new DataView(bytes.buffer, bytes.byteOffset, 8).getBigUint64(0, true);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
async function getTableFieldPos(tablePos, fieldIndex) {
|
|
251
|
+
const vtableRelative = await readInt32(tablePos);
|
|
252
|
+
const vtablePos = tablePos - vtableRelative;
|
|
253
|
+
const vtableLength = await readUint16(vtablePos);
|
|
254
|
+
const fieldPos = vtablePos + 4 + fieldIndex * 2;
|
|
255
|
+
if (fieldPos + 2 > vtablePos + vtableLength) {
|
|
256
|
+
return null;
|
|
257
|
+
}
|
|
258
|
+
const fieldOffset = await readUint16(fieldPos);
|
|
259
|
+
return fieldOffset === 0 ? null : tablePos + fieldOffset;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
async function readStringAtField(tablePos, fieldIndex) {
|
|
263
|
+
const fieldPos = await getTableFieldPos(tablePos, fieldIndex);
|
|
264
|
+
if (fieldPos == null) {
|
|
265
|
+
return null;
|
|
266
|
+
}
|
|
267
|
+
const stringPos = fieldPos + await readInt32(fieldPos);
|
|
268
|
+
const stringLength = await readInt32(stringPos);
|
|
269
|
+
const bytes = await readSlice(stringPos + 4, stringLength);
|
|
270
|
+
return TEXT_DECODER.decode(bytes);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
async function readInt32VectorAtField(tablePos, fieldIndex) {
|
|
274
|
+
const fieldPos = await getTableFieldPos(tablePos, fieldIndex);
|
|
275
|
+
if (fieldPos == null) {
|
|
276
|
+
return [];
|
|
277
|
+
}
|
|
278
|
+
const vectorPos = fieldPos + await readInt32(fieldPos);
|
|
279
|
+
const length = await readInt32(vectorPos);
|
|
280
|
+
const values = [];
|
|
281
|
+
for (let index = 0; index < length; index++) {
|
|
282
|
+
values.push(await readInt32(vectorPos + 4 + index * 4));
|
|
283
|
+
}
|
|
284
|
+
return values;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
async function readFloat32VectorAtField(tablePos, fieldIndex) {
|
|
288
|
+
const fieldPos = await getTableFieldPos(tablePos, fieldIndex);
|
|
289
|
+
if (fieldPos == null) {
|
|
290
|
+
return [];
|
|
291
|
+
}
|
|
292
|
+
const vectorPos = fieldPos + await readInt32(fieldPos);
|
|
293
|
+
const length = await readInt32(vectorPos);
|
|
294
|
+
const values = [];
|
|
295
|
+
for (let index = 0; index < length; index++) {
|
|
296
|
+
values.push(await readFloat32(vectorPos + 4 + index * 4));
|
|
297
|
+
}
|
|
298
|
+
return values;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
async function readInt64VectorAtField(tablePos, fieldIndex) {
|
|
302
|
+
const fieldPos = await getTableFieldPos(tablePos, fieldIndex);
|
|
303
|
+
if (fieldPos == null) {
|
|
304
|
+
return [];
|
|
305
|
+
}
|
|
306
|
+
const vectorPos = fieldPos + await readInt32(fieldPos);
|
|
307
|
+
const length = await readInt32(vectorPos);
|
|
308
|
+
const values = [];
|
|
309
|
+
for (let index = 0; index < length; index++) {
|
|
310
|
+
const value = await readBigInt64(vectorPos + 4 + index * 8);
|
|
311
|
+
const numeric = Number(value);
|
|
312
|
+
if (!Number.isSafeInteger(numeric)) {
|
|
313
|
+
throw new Error(
|
|
314
|
+
`TFLite parser: int64 quantization value ${String(value)} exceeds JS safe integer range.`
|
|
315
|
+
);
|
|
316
|
+
}
|
|
317
|
+
values.push(numeric);
|
|
318
|
+
}
|
|
319
|
+
return values;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
async function readTableVectorLength(tablePos, fieldIndex) {
|
|
323
|
+
const fieldPos = await getTableFieldPos(tablePos, fieldIndex);
|
|
324
|
+
if (fieldPos == null) {
|
|
325
|
+
return 0;
|
|
326
|
+
}
|
|
327
|
+
const vectorPos = fieldPos + await readInt32(fieldPos);
|
|
328
|
+
return readInt32(vectorPos);
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
async function readTableVectorEntry(tablePos, fieldIndex, index) {
|
|
332
|
+
const fieldPos = await getTableFieldPos(tablePos, fieldIndex);
|
|
333
|
+
if (fieldPos == null) {
|
|
334
|
+
return null;
|
|
335
|
+
}
|
|
336
|
+
const vectorPos = fieldPos + await readInt32(fieldPos);
|
|
337
|
+
const length = await readInt32(vectorPos);
|
|
338
|
+
if (index < 0 || index >= length) {
|
|
339
|
+
return null;
|
|
340
|
+
}
|
|
341
|
+
const entryPos = vectorPos + 4 + index * 4;
|
|
342
|
+
return entryPos + await readInt32(entryPos);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
async function readByteVectorLocationAtField(tablePos, fieldIndex) {
|
|
346
|
+
const fieldPos = await getTableFieldPos(tablePos, fieldIndex);
|
|
347
|
+
if (fieldPos == null) {
|
|
348
|
+
return null;
|
|
349
|
+
}
|
|
350
|
+
const vectorPos = fieldPos + await readInt32(fieldPos);
|
|
351
|
+
return {
|
|
352
|
+
offset: vectorPos + 4,
|
|
353
|
+
size: await readInt32(vectorPos),
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
return {
|
|
358
|
+
label,
|
|
359
|
+
size,
|
|
360
|
+
readSlice,
|
|
361
|
+
readInt32,
|
|
362
|
+
readUint32,
|
|
363
|
+
readUint16,
|
|
364
|
+
readUint8,
|
|
365
|
+
readFloat32,
|
|
366
|
+
readBigInt64,
|
|
367
|
+
readBigUint64,
|
|
368
|
+
getTableFieldPos,
|
|
369
|
+
readStringAtField,
|
|
370
|
+
readInt32VectorAtField,
|
|
371
|
+
readFloat32VectorAtField,
|
|
372
|
+
readInt64VectorAtField,
|
|
373
|
+
readTableVectorLength,
|
|
374
|
+
readTableVectorEntry,
|
|
375
|
+
readByteVectorLocationAtField,
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
async function readTensorQuantization(reader, tensorPos, tensorName, sourceDtype) {
|
|
380
|
+
const quantizationPos = await reader.getTableFieldPos(tensorPos, 4);
|
|
381
|
+
if (quantizationPos == null) {
|
|
382
|
+
throw new Error(
|
|
383
|
+
`TFLite parser: quantized tensor "${tensorName}" is missing quantization parameters. ` +
|
|
384
|
+
`Packed LiteRT-LM companion tensors such as "${tensorName}.sum_i" and ` +
|
|
385
|
+
`"${tensorName}_quantized_scale" are not supported in direct-source mode yet.`
|
|
386
|
+
);
|
|
387
|
+
}
|
|
388
|
+
const tablePos = quantizationPos + await reader.readInt32(quantizationPos);
|
|
389
|
+
const scales = await reader.readFloat32VectorAtField(tablePos, 2);
|
|
390
|
+
const zeroPoints = await reader.readInt64VectorAtField(tablePos, 3);
|
|
391
|
+
const quantizedDimensionFieldPos = await reader.getTableFieldPos(tablePos, 5);
|
|
392
|
+
const quantizedDimension = quantizedDimensionFieldPos == null
|
|
393
|
+
? 0
|
|
394
|
+
: await reader.readInt32(quantizedDimensionFieldPos);
|
|
395
|
+
|
|
396
|
+
if (scales.length !== 1) {
|
|
397
|
+
throw new Error(
|
|
398
|
+
`TFLite parser: quantized tensor "${tensorName}" must use exactly one scale value. ` +
|
|
399
|
+
`Found ${scales.length}. Per-channel quantization is not supported in direct-source mode.`
|
|
400
|
+
);
|
|
401
|
+
}
|
|
402
|
+
if (zeroPoints.length > 1) {
|
|
403
|
+
throw new Error(
|
|
404
|
+
`TFLite parser: quantized tensor "${tensorName}" must use at most one zero_point value. ` +
|
|
405
|
+
`Found ${zeroPoints.length}. Per-channel quantization is not supported in direct-source mode.`
|
|
406
|
+
);
|
|
407
|
+
}
|
|
408
|
+
if (quantizedDimension !== 0) {
|
|
409
|
+
throw new Error(
|
|
410
|
+
`TFLite parser: quantized tensor "${tensorName}" uses quantized_dimension=${quantizedDimension}. ` +
|
|
411
|
+
'Only per-tensor affine quantization is supported in direct-source mode.'
|
|
412
|
+
);
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
const scale = Number(scales[0]);
|
|
416
|
+
if (!Number.isFinite(scale) || scale <= 0) {
|
|
417
|
+
throw new Error(
|
|
418
|
+
`TFLite parser: quantized tensor "${tensorName}" has invalid scale ${scales[0]}.`
|
|
419
|
+
);
|
|
420
|
+
}
|
|
421
|
+
const zeroPoint = zeroPoints.length === 0 ? 0 : Number(zeroPoints[0]);
|
|
422
|
+
if (!Number.isSafeInteger(zeroPoint)) {
|
|
423
|
+
throw new Error(
|
|
424
|
+
`TFLite parser: quantized tensor "${tensorName}" has invalid zero_point ${zeroPoints[0]}.`
|
|
425
|
+
);
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
return {
|
|
429
|
+
kind: 'affine_dequant',
|
|
430
|
+
scheme: 'per_tensor_affine',
|
|
431
|
+
sourceDtype,
|
|
432
|
+
targetDtype: 'F16',
|
|
433
|
+
scale,
|
|
434
|
+
zeroPoint,
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
async function readBufferLocation(reader, bufferPos, tensorName) {
|
|
439
|
+
const inlineLocation = await reader.readByteVectorLocationAtField(bufferPos, 0);
|
|
440
|
+
if (inlineLocation && inlineLocation.size > 0) {
|
|
441
|
+
return inlineLocation;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
const offsetFieldPos = await reader.getTableFieldPos(bufferPos, 1);
|
|
445
|
+
const sizeFieldPos = await reader.getTableFieldPos(bufferPos, 2);
|
|
446
|
+
if (offsetFieldPos == null || sizeFieldPos == null) {
|
|
447
|
+
return null;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
const offsetRaw = await reader.readBigUint64(offsetFieldPos);
|
|
451
|
+
const sizeRaw = await reader.readBigUint64(sizeFieldPos);
|
|
452
|
+
const offset = Number(offsetRaw);
|
|
453
|
+
const size = Number(sizeRaw);
|
|
454
|
+
if (!Number.isSafeInteger(offset) || offset < 0) {
|
|
455
|
+
throw new Error(
|
|
456
|
+
`TFLite parser: tensor "${tensorName}" uses buffer offset ${String(offsetRaw)} outside JS safe integer range.`
|
|
457
|
+
);
|
|
458
|
+
}
|
|
459
|
+
if (!Number.isSafeInteger(size) || size < 0) {
|
|
460
|
+
throw new Error(
|
|
461
|
+
`TFLite parser: tensor "${tensorName}" uses buffer size ${String(sizeRaw)} outside JS safe integer range.`
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
if (offset + size > reader.size) {
|
|
465
|
+
throw new Error(
|
|
466
|
+
`TFLite parser: tensor "${tensorName}" buffer range exceeds file bounds (offset=${offset}, size=${size}, fileSize=${reader.size}).`
|
|
467
|
+
);
|
|
468
|
+
}
|
|
469
|
+
return size > 0 ? { offset, size } : null;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
async function parseTFLiteReader(reader, options = {}) {
|
|
473
|
+
const parseOptions = normalizeParseOptions(options);
|
|
474
|
+
const rootOffset = await reader.readInt32(0);
|
|
475
|
+
const fileIdentifier = decodeAscii(await reader.readSlice(4, 4));
|
|
476
|
+
if (fileIdentifier !== TFLITE_FILE_IDENTIFIER) {
|
|
477
|
+
throw new Error(
|
|
478
|
+
`TFLite parser: invalid file identifier "${fileIdentifier || '(empty)'}". ` +
|
|
479
|
+
`Expected "${TFLITE_FILE_IDENTIFIER}".`
|
|
480
|
+
);
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
const modelPos = rootOffset;
|
|
484
|
+
const versionFieldPos = await reader.getTableFieldPos(modelPos, 0);
|
|
485
|
+
const schemaVersion = versionFieldPos == null ? 0 : await reader.readUint32(versionFieldPos);
|
|
486
|
+
const description = await reader.readStringAtField(modelPos, 3);
|
|
487
|
+
const subgraphCount = await reader.readTableVectorLength(modelPos, 2);
|
|
488
|
+
const bufferCount = await reader.readTableVectorLength(modelPos, 4);
|
|
489
|
+
|
|
490
|
+
if (subgraphCount < 1) {
|
|
491
|
+
throw new Error('TFLite parser: model must contain at least one subgraph.');
|
|
492
|
+
}
|
|
493
|
+
if (bufferCount < 1) {
|
|
494
|
+
throw new Error('TFLite parser: model must contain at least one buffer.');
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
const mainSubgraphPos = await reader.readTableVectorEntry(modelPos, 2, 0);
|
|
498
|
+
if (mainSubgraphPos == null) {
|
|
499
|
+
throw new Error('TFLite parser: failed to resolve the main subgraph.');
|
|
500
|
+
}
|
|
501
|
+
const mainSubgraphName = await reader.readStringAtField(mainSubgraphPos, 4);
|
|
502
|
+
const tensorCount = await reader.readTableVectorLength(mainSubgraphPos, 0);
|
|
503
|
+
const tensors = [];
|
|
504
|
+
const seenNames = new Set();
|
|
505
|
+
|
|
506
|
+
for (let index = 0; index < tensorCount; index++) {
|
|
507
|
+
const tensorPos = await reader.readTableVectorEntry(mainSubgraphPos, 0, index);
|
|
508
|
+
if (tensorPos == null) {
|
|
509
|
+
continue;
|
|
510
|
+
}
|
|
511
|
+
const name = await reader.readStringAtField(tensorPos, 3);
|
|
512
|
+
if (!name) {
|
|
513
|
+
continue;
|
|
514
|
+
}
|
|
515
|
+
if (seenNames.has(name)) {
|
|
516
|
+
throw new Error(`TFLite parser: duplicate tensor name "${name}" in the main subgraph.`);
|
|
517
|
+
}
|
|
518
|
+
seenNames.add(name);
|
|
519
|
+
|
|
520
|
+
const bufferFieldPos = await reader.getTableFieldPos(tensorPos, 2);
|
|
521
|
+
const bufferIndex = bufferFieldPos == null ? 0 : await reader.readUint32(bufferFieldPos);
|
|
522
|
+
if (bufferIndex === 0) {
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
if (bufferIndex >= bufferCount) {
|
|
526
|
+
throw new Error(
|
|
527
|
+
`TFLite parser: tensor "${name}" references buffer ${bufferIndex}, ` +
|
|
528
|
+
`but the model only has ${bufferCount} buffers.`
|
|
529
|
+
);
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
const bufferPos = await reader.readTableVectorEntry(modelPos, 4, bufferIndex);
|
|
533
|
+
if (bufferPos == null) {
|
|
534
|
+
throw new Error(`TFLite parser: missing buffer table entry ${bufferIndex} for tensor "${name}".`);
|
|
535
|
+
}
|
|
536
|
+
const dataLocation = await readBufferLocation(reader, bufferPos, name);
|
|
537
|
+
if (!dataLocation || dataLocation.size <= 0) {
|
|
538
|
+
continue;
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
const typeFieldPos = await reader.getTableFieldPos(tensorPos, 1);
|
|
542
|
+
const dtypeId = typeFieldPos == null ? TFLITE_TENSOR_TYPE.FLOAT32 : await reader.readUint8(typeFieldPos);
|
|
543
|
+
const dtypeName = TFLITE_TENSOR_TYPE_NAME[dtypeId] ?? `UNKNOWN_${dtypeId}`;
|
|
544
|
+
const mappedSourceDtype = TFLITE_TENSOR_SOURCE_DTYPE_MAP[dtypeId] ?? null;
|
|
545
|
+
const sourceDtype = mappedSourceDtype ?? (parseOptions.allowPackedQuantization ? dtypeName : null);
|
|
546
|
+
if (!sourceDtype) {
|
|
547
|
+
throw new Error(`TFLite parser: tensor "${name}" uses unsupported constant dtype ${dtypeName}.`);
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
const isVariableFieldPos = await reader.getTableFieldPos(tensorPos, 5);
|
|
551
|
+
const isVariable = isVariableFieldPos == null ? false : await reader.readUint8(isVariableFieldPos) !== 0;
|
|
552
|
+
if (isVariable) {
|
|
553
|
+
throw new Error(
|
|
554
|
+
`TFLite parser: variable tensor "${name}" is not supported in direct-source mode.`
|
|
555
|
+
);
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
const shape = await reader.readInt32VectorAtField(tensorPos, 0);
|
|
559
|
+
const elementCount = computeTensorElementCount(shape, name);
|
|
560
|
+
const isQuantizedSourceDtype = (
|
|
561
|
+
mappedSourceDtype === 'INT8'
|
|
562
|
+
|| mappedSourceDtype === 'UINT8'
|
|
563
|
+
|| mappedSourceDtype === 'INT4'
|
|
564
|
+
);
|
|
565
|
+
let sourceTransform = null;
|
|
566
|
+
if (isQuantizedSourceDtype) {
|
|
567
|
+
try {
|
|
568
|
+
sourceTransform = await readTensorQuantization(reader, tensorPos, name, mappedSourceDtype);
|
|
569
|
+
} catch (error) {
|
|
570
|
+
if (!parseOptions.allowPackedQuantization) {
|
|
571
|
+
throw error;
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
const dtype = sourceTransform?.targetDtype
|
|
576
|
+
?? (TFLITE_TENSOR_DTYPE_MAP[dtypeId] ?? (parseOptions.allowPackedQuantization ? sourceDtype : null));
|
|
577
|
+
if (!dtype && !parseOptions.allowPackedQuantization) {
|
|
578
|
+
throw new Error(
|
|
579
|
+
`TFLite parser: tensor "${name}" uses unsupported constant dtype ${dtypeName}.`
|
|
580
|
+
);
|
|
581
|
+
}
|
|
582
|
+
const expectedBytes = mappedSourceDtype
|
|
583
|
+
? computeTFLiteSourceByteSize(elementCount, mappedSourceDtype, name)
|
|
584
|
+
: null;
|
|
585
|
+
if (expectedBytes !== null && expectedBytes !== dataLocation.size && !parseOptions.allowPackedQuantization) {
|
|
586
|
+
throw new Error(
|
|
587
|
+
`TFLite parser: tensor "${name}" byte size mismatch. ` +
|
|
588
|
+
`Expected ${expectedBytes} bytes from shape ${JSON.stringify(shape)} and dtype ${sourceDtype}, ` +
|
|
589
|
+
`found ${dataLocation.size}.`
|
|
590
|
+
);
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
tensors.push({
|
|
594
|
+
name,
|
|
595
|
+
shape,
|
|
596
|
+
dtype: dtype ?? sourceDtype,
|
|
597
|
+
dtypeId,
|
|
598
|
+
sourceDtype,
|
|
599
|
+
offset: dataLocation.offset,
|
|
600
|
+
size: dataLocation.size,
|
|
601
|
+
buffer: bufferIndex,
|
|
602
|
+
subgraphIndex: 0,
|
|
603
|
+
isVariable: false,
|
|
604
|
+
...(sourceTransform ? { sourceTransform } : {}),
|
|
605
|
+
});
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
if (tensors.length === 0) {
|
|
609
|
+
throw new Error(
|
|
610
|
+
'TFLite parser: no supported constant tensors were found in the main subgraph.'
|
|
611
|
+
);
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
return {
|
|
615
|
+
schemaVersion,
|
|
616
|
+
description: description || null,
|
|
617
|
+
subgraphCount,
|
|
618
|
+
mainSubgraphName: mainSubgraphName || null,
|
|
619
|
+
tensors,
|
|
620
|
+
sourceQuantization: resolveDirectSourceQuantization(tensors),
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
export async function parseTFLite(buffer, options = {}) {
|
|
625
|
+
return parseTFLiteReader(
|
|
626
|
+
createPagedSourceReader(createMemoryRangeSource(buffer)),
|
|
627
|
+
options
|
|
628
|
+
);
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
export async function parseTFLiteFromSource(source, options = {}) {
|
|
632
|
+
return parseTFLiteReader(createPagedSourceReader(source), options);
|
|
633
|
+
}
|
|
@@ -1,18 +1,20 @@
|
|
|
1
1
|
export { InferencePipeline, EmbeddingPipeline, createPipeline } from '../inference/pipelines/text.js';
|
|
2
|
-
export {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
2
|
+
export type {
|
|
3
|
+
GenerateOptions,
|
|
4
|
+
GenerationResult,
|
|
5
|
+
KVCacheSnapshot,
|
|
6
|
+
LayerWeights,
|
|
7
|
+
ExpertWeights,
|
|
8
|
+
RouterWeights,
|
|
9
|
+
PipelineContexts,
|
|
10
|
+
LogitsStepResult,
|
|
11
|
+
PrefillResult,
|
|
12
|
+
PrefillEmbeddingResult,
|
|
13
|
+
AdvanceEmbeddingResult,
|
|
14
|
+
PipelineStats,
|
|
15
|
+
BatchingStats,
|
|
16
|
+
PromptInput,
|
|
17
|
+
} from '../inference/pipelines/text.js';
|
|
14
18
|
export type { ParsedModelConfig } from '../inference/pipelines/text/config.js';
|
|
15
19
|
export type { SamplingOptions } from '../inference/pipelines/text/sampling.js';
|
|
16
|
-
export type { InferencePipeline, PipelineContexts, KVCacheSnapshot, LayerWeights, ExpertWeights, RouterWeights } from '../inference/pipelines/text.js';
|
|
17
|
-
export type { GenerateOptions, GenerationResult } from '../inference/pipelines/text.js';
|
|
18
20
|
export type { LoRAAdapter, LoRAModuleName } from '../inference/pipelines/text/lora-types.js';
|
package/src/generation/index.js
CHANGED
|
@@ -1,32 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import { log } from '../debug/index.js';
|
|
3
|
-
|
|
4
|
-
export { InferencePipeline, EmbeddingPipeline } from '../inference/pipelines/text.js';
|
|
5
|
-
export { parseModelConfig, parseModelConfigFromManifest } from '../inference/pipelines/text/config.js';
|
|
6
|
-
export { loadWeights, initTokenizer, isStopToken } from '../inference/pipelines/text/init.js';
|
|
7
|
-
export { initTokenizerFromManifest } from '../inference/pipelines/text/model-load.js';
|
|
8
|
-
export {
|
|
9
|
-
StructuredJsonHeadPipeline,
|
|
10
|
-
isStructuredJsonHeadModelType,
|
|
11
|
-
createStructuredJsonHeadPipeline,
|
|
12
|
-
DreamStructuredPipeline,
|
|
13
|
-
isDreamStructuredModelType,
|
|
14
|
-
createDreamStructuredPipeline,
|
|
15
|
-
} from '../inference/pipelines/structured/json-head-pipeline.js';
|
|
16
|
-
|
|
17
|
-
/** Required methods that any pipeline returned from createPipeline must implement. */
|
|
18
|
-
const REQUIRED_PIPELINE_METHODS = ['generate', 'loadModel', 'reset', 'unload'];
|
|
19
|
-
|
|
20
|
-
export async function createPipeline(manifest, contexts = {}) {
|
|
21
|
-
const pipeline = await _createPipeline(manifest, contexts);
|
|
22
|
-
for (const method of REQUIRED_PIPELINE_METHODS) {
|
|
23
|
-
if (typeof pipeline[method] !== 'function') {
|
|
24
|
-
log.warn(
|
|
25
|
-
'Generation',
|
|
26
|
-
`Created pipeline for modelType "${manifest?.modelType}" is missing required method "${method}". ` +
|
|
27
|
-
'This may cause failures during generation.'
|
|
28
|
-
);
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
return pipeline;
|
|
32
|
-
}
|
|
1
|
+
export { InferencePipeline, EmbeddingPipeline, createPipeline } from '../inference/pipelines/text.js';
|