doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -15,6 +15,7 @@ import { parseSafetensorsHeader } from '../formats/safetensors/types.js';
|
|
|
15
15
|
import { parseTFLiteFromSource } from '../formats/tflite/types.js';
|
|
16
16
|
import { log } from '../debug/index.js';
|
|
17
17
|
import { formatBytes } from '../storage/quota.js';
|
|
18
|
+
import { toArrayBuffer } from '../utils/array-buffer.js';
|
|
18
19
|
import {
|
|
19
20
|
createSourceStorageContext,
|
|
20
21
|
} from './source-runtime-bundle.js';
|
|
@@ -30,16 +31,6 @@ import {
|
|
|
30
31
|
} from './litert-package-runtime.js';
|
|
31
32
|
const MAX_NODE_READ_BYTES = 64 * 1024 * 1024;
|
|
32
33
|
|
|
33
|
-
function toArrayBuffer(value, label) {
|
|
34
|
-
if (value instanceof ArrayBuffer) {
|
|
35
|
-
return value;
|
|
36
|
-
}
|
|
37
|
-
if (value instanceof Uint8Array) {
|
|
38
|
-
return value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength);
|
|
39
|
-
}
|
|
40
|
-
throw new Error(`${label} must return ArrayBuffer or Uint8Array.`);
|
|
41
|
-
}
|
|
42
|
-
|
|
43
34
|
function normalizePath(value) {
|
|
44
35
|
return String(value || '').trim();
|
|
45
36
|
}
|
|
@@ -121,14 +112,107 @@ async function readRange(filePath, offset, length) {
|
|
|
121
112
|
}
|
|
122
113
|
}
|
|
123
114
|
|
|
124
|
-
|
|
125
|
-
const
|
|
115
|
+
function createNodeFileAccess() {
|
|
116
|
+
const readers = new Map();
|
|
117
|
+
|
|
118
|
+
const getReader = (filePath) => {
|
|
119
|
+
const normalizedPath = normalizePath(filePath);
|
|
120
|
+
if (!normalizedPath) {
|
|
121
|
+
throw new Error('node source runtime: filePath is required.');
|
|
122
|
+
}
|
|
123
|
+
let reader = readers.get(normalizedPath);
|
|
124
|
+
if (reader) {
|
|
125
|
+
return reader;
|
|
126
|
+
}
|
|
127
|
+
let handlePromise = null;
|
|
128
|
+
let sizePromise = null;
|
|
129
|
+
let closed = false;
|
|
130
|
+
const ensureHandle = async () => {
|
|
131
|
+
if (closed) {
|
|
132
|
+
throw new Error(`node source runtime: file reader already closed for "${normalizedPath}".`);
|
|
133
|
+
}
|
|
134
|
+
if (!handlePromise) {
|
|
135
|
+
handlePromise = fs.open(normalizedPath, 'r').catch((error) => {
|
|
136
|
+
handlePromise = null;
|
|
137
|
+
throw error;
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
return handlePromise;
|
|
141
|
+
};
|
|
142
|
+
const getSize = async () => {
|
|
143
|
+
if (!sizePromise) {
|
|
144
|
+
sizePromise = (async () => {
|
|
145
|
+
const handle = await ensureHandle();
|
|
146
|
+
const stats = await handle.stat();
|
|
147
|
+
return Number(stats.size);
|
|
148
|
+
})().catch((error) => {
|
|
149
|
+
sizePromise = null;
|
|
150
|
+
throw error;
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
return sizePromise;
|
|
154
|
+
};
|
|
155
|
+
reader = {
|
|
156
|
+
async readRange(offset, length) {
|
|
157
|
+
if (!Number.isFinite(offset) || !Number.isFinite(length) || length <= 0) {
|
|
158
|
+
return new ArrayBuffer(0);
|
|
159
|
+
}
|
|
160
|
+
const start = Math.max(0, Math.floor(offset));
|
|
161
|
+
const fileSize = await getSize();
|
|
162
|
+
const end = Math.min(fileSize, start + Math.floor(length));
|
|
163
|
+
if (end <= start) {
|
|
164
|
+
return new ArrayBuffer(0);
|
|
165
|
+
}
|
|
166
|
+
const handle = await ensureHandle();
|
|
167
|
+
const out = Buffer.allocUnsafe(end - start);
|
|
168
|
+
let pos = 0;
|
|
169
|
+
while (pos < out.length) {
|
|
170
|
+
const nextChunkBytes = Math.min(out.length - pos, MAX_NODE_READ_BYTES);
|
|
171
|
+
const { bytesRead } = await handle.read(out, pos, nextChunkBytes, start + pos);
|
|
172
|
+
if (bytesRead === 0) break;
|
|
173
|
+
pos += bytesRead;
|
|
174
|
+
}
|
|
175
|
+
return out.buffer.slice(out.byteOffset, out.byteOffset + out.byteLength);
|
|
176
|
+
},
|
|
177
|
+
async getSize() {
|
|
178
|
+
return getSize();
|
|
179
|
+
},
|
|
180
|
+
async close() {
|
|
181
|
+
closed = true;
|
|
182
|
+
const handle = await handlePromise;
|
|
183
|
+
handlePromise = null;
|
|
184
|
+
if (handle) {
|
|
185
|
+
await handle.close();
|
|
186
|
+
}
|
|
187
|
+
},
|
|
188
|
+
};
|
|
189
|
+
readers.set(normalizedPath, reader);
|
|
190
|
+
return reader;
|
|
191
|
+
};
|
|
192
|
+
|
|
193
|
+
return {
|
|
194
|
+
async readRange(filePath, offset, length) {
|
|
195
|
+
return getReader(filePath).readRange(offset, length);
|
|
196
|
+
},
|
|
197
|
+
async getSize(filePath) {
|
|
198
|
+
return getReader(filePath).getSize();
|
|
199
|
+
},
|
|
200
|
+
async close() {
|
|
201
|
+
const pending = Array.from(readers.values(), (reader) => reader.close());
|
|
202
|
+
readers.clear();
|
|
203
|
+
await Promise.all(pending);
|
|
204
|
+
},
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
async function readSafetensorsHeaderFromFile(filePath, fileAccess) {
|
|
209
|
+
const headerPrefixBuffer = await fileAccess.readRange(filePath, 0, 8);
|
|
126
210
|
const prefixBytes = new Uint8Array(headerPrefixBuffer);
|
|
127
211
|
if (prefixBytes.byteLength < 8) {
|
|
128
212
|
throw new Error(`Invalid safetensors header prefix for "${filePath}"`);
|
|
129
213
|
}
|
|
130
214
|
const headerSize = Number(new DataView(headerPrefixBuffer).getBigUint64(0, true));
|
|
131
|
-
const headerBuffer = await readRange(filePath, 8, headerSize);
|
|
215
|
+
const headerBuffer = await fileAccess.readRange(filePath, 8, headerSize);
|
|
132
216
|
const fullHeader = new Uint8Array(8 + headerSize);
|
|
133
217
|
fullHeader.set(prefixBytes, 0);
|
|
134
218
|
fullHeader.set(new Uint8Array(headerBuffer), 8);
|
|
@@ -237,7 +321,7 @@ function applyPackageTokenizerAssets(parsedArtifact, tokenizerAssets) {
|
|
|
237
321
|
return next;
|
|
238
322
|
}
|
|
239
323
|
|
|
240
|
-
async function parseSafetensorsInput(inputDir) {
|
|
324
|
+
async function parseSafetensorsInput(inputDir, fileAccess) {
|
|
241
325
|
const configPath = path.join(inputDir, 'config.json');
|
|
242
326
|
if (!(await fileExists(configPath))) {
|
|
243
327
|
return null;
|
|
@@ -257,7 +341,7 @@ async function parseSafetensorsInput(inputDir) {
|
|
|
257
341
|
},
|
|
258
342
|
async loadSingleSafetensors(suffix) {
|
|
259
343
|
const filePath = path.join(inputDir, suffix);
|
|
260
|
-
const parsed = await readSafetensorsHeaderFromFile(filePath);
|
|
344
|
+
const parsed = await readSafetensorsHeaderFromFile(filePath, fileAccess);
|
|
261
345
|
return parsed.tensors.map((tensor) => ({
|
|
262
346
|
...tensor,
|
|
263
347
|
sourcePath: filePath,
|
|
@@ -268,7 +352,7 @@ async function parseSafetensorsInput(inputDir) {
|
|
|
268
352
|
const tensors = [];
|
|
269
353
|
for (const shardFile of shardFiles) {
|
|
270
354
|
const shardPath = path.join(inputDir, shardFile);
|
|
271
|
-
const parsed = await readSafetensorsHeaderFromFile(shardPath);
|
|
355
|
+
const parsed = await readSafetensorsHeaderFromFile(shardPath, fileAccess);
|
|
272
356
|
for (const tensor of parsed.tensors) {
|
|
273
357
|
tensors.push({
|
|
274
358
|
...tensor,
|
|
@@ -328,7 +412,7 @@ async function parseSafetensorsInput(inputDir) {
|
|
|
328
412
|
};
|
|
329
413
|
}
|
|
330
414
|
|
|
331
|
-
async function parseTfliteInput(tflitePath) {
|
|
415
|
+
async function parseTfliteInput(tflitePath, fileAccess) {
|
|
332
416
|
const tfliteStats = await getPathStats(tflitePath, 'TFLite file');
|
|
333
417
|
const inputDir = path.dirname(tflitePath);
|
|
334
418
|
const configPath = path.join(inputDir, 'config.json');
|
|
@@ -343,7 +427,7 @@ async function parseTfliteInput(tflitePath) {
|
|
|
343
427
|
name: path.basename(tflitePath),
|
|
344
428
|
size: Number(tfliteStats.size),
|
|
345
429
|
async readRange(offset, length) {
|
|
346
|
-
return readRange(tflitePath, offset, length);
|
|
430
|
+
return fileAccess.readRange(tflitePath, offset, length);
|
|
347
431
|
},
|
|
348
432
|
});
|
|
349
433
|
const tokenizerAssets = await collectTokenizerAssets(inputDir);
|
|
@@ -379,7 +463,7 @@ async function parseTfliteInput(tflitePath) {
|
|
|
379
463
|
};
|
|
380
464
|
}
|
|
381
465
|
|
|
382
|
-
function createNodeVirtualFileReaders(packagePath, virtualFiles) {
|
|
466
|
+
function createNodeVirtualFileReaders(packagePath, virtualFiles, fileAccess) {
|
|
383
467
|
const virtualFileMap = new Map(
|
|
384
468
|
(Array.isArray(virtualFiles) ? virtualFiles : []).map((entry) => [entry.path, entry])
|
|
385
469
|
);
|
|
@@ -396,7 +480,7 @@ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
|
|
|
396
480
|
const readRangeFromVirtualFile = async (virtualPath, offset, length) => {
|
|
397
481
|
const entry = resolveVirtualFile(virtualPath);
|
|
398
482
|
if (entry.externalPath) {
|
|
399
|
-
return readRange(entry.externalPath, offset, length);
|
|
483
|
+
return fileAccess.readRange(entry.externalPath, offset, length);
|
|
400
484
|
}
|
|
401
485
|
const start = Math.max(0, Math.floor(Number(offset) || 0));
|
|
402
486
|
const requested = Math.max(0, Math.floor(Number(length) || 0));
|
|
@@ -405,7 +489,7 @@ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
|
|
|
405
489
|
if (readLength <= 0) {
|
|
406
490
|
return new ArrayBuffer(0);
|
|
407
491
|
}
|
|
408
|
-
return readRange(packagePath, entry.offset + start, readLength);
|
|
492
|
+
return fileAccess.readRange(packagePath, entry.offset + start, readLength);
|
|
409
493
|
};
|
|
410
494
|
|
|
411
495
|
const streamRange = async function* (virtualPath, offset, length, options = {}) {
|
|
@@ -458,7 +542,7 @@ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
|
|
|
458
542
|
const entry = resolveVirtualFile(virtualPath);
|
|
459
543
|
const bytes = entry.externalPath
|
|
460
544
|
? await fs.readFile(entry.externalPath)
|
|
461
|
-
: await readRange(packagePath, entry.offset, entry.size);
|
|
545
|
+
: await fileAccess.readRange(packagePath, entry.offset, entry.size);
|
|
462
546
|
return new TextDecoder().decode(bytes);
|
|
463
547
|
};
|
|
464
548
|
|
|
@@ -466,7 +550,7 @@ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
|
|
|
466
550
|
const entry = resolveVirtualFile(virtualPath);
|
|
467
551
|
return entry.externalPath
|
|
468
552
|
? fs.readFile(entry.externalPath)
|
|
469
|
-
: readRange(packagePath, entry.offset, entry.size);
|
|
553
|
+
: fileAccess.readRange(packagePath, entry.offset, entry.size);
|
|
470
554
|
};
|
|
471
555
|
|
|
472
556
|
return {
|
|
@@ -475,11 +559,12 @@ function createNodeVirtualFileReaders(packagePath, virtualFiles) {
|
|
|
475
559
|
streamRange,
|
|
476
560
|
readText,
|
|
477
561
|
readBinary,
|
|
562
|
+
close: fileAccess.close,
|
|
478
563
|
};
|
|
479
564
|
}
|
|
480
565
|
|
|
481
|
-
async function addHashesToVirtualEntries(packagePath, virtualFiles, entries, hashAlgorithm) {
|
|
482
|
-
const readers = createNodeVirtualFileReaders(packagePath, virtualFiles);
|
|
566
|
+
async function addHashesToVirtualEntries(packagePath, virtualFiles, fileAccess, entries, hashAlgorithm) {
|
|
567
|
+
const readers = createNodeVirtualFileReaders(packagePath, virtualFiles, fileAccess);
|
|
483
568
|
const hashedEntries = [];
|
|
484
569
|
for (const entry of Array.isArray(entries) ? entries : []) {
|
|
485
570
|
const virtualPath = normalizePath(entry?.path);
|
|
@@ -505,7 +590,7 @@ async function addHashesToVirtualEntries(packagePath, virtualFiles, entries, has
|
|
|
505
590
|
return hashedEntries;
|
|
506
591
|
}
|
|
507
592
|
|
|
508
|
-
async function parseLiteRTPackageInput(packagePath, sourceKind) {
|
|
593
|
+
async function parseLiteRTPackageInput(packagePath, sourceKind, fileAccess) {
|
|
509
594
|
const stats = await getPathStats(packagePath, `LiteRT package (${sourceKind})`);
|
|
510
595
|
const resolved = await resolveLiteRTPackageParsedArtifact({
|
|
511
596
|
sourceKind,
|
|
@@ -514,7 +599,7 @@ async function parseLiteRTPackageInput(packagePath, sourceKind) {
|
|
|
514
599
|
name: path.basename(packagePath),
|
|
515
600
|
size: Number(stats.size),
|
|
516
601
|
async readRange(offset, length) {
|
|
517
|
-
return readRange(packagePath, offset, length);
|
|
602
|
+
return fileAccess.readRange(packagePath, offset, length);
|
|
518
603
|
},
|
|
519
604
|
},
|
|
520
605
|
});
|
|
@@ -527,14 +612,14 @@ async function parseLiteRTPackageInput(packagePath, sourceKind) {
|
|
|
527
612
|
return {
|
|
528
613
|
...parsedArtifact,
|
|
529
614
|
sourceRoot: packagePath,
|
|
530
|
-
storageReaders: createNodeVirtualFileReaders(packagePath, virtualFiles),
|
|
615
|
+
storageReaders: createNodeVirtualFileReaders(packagePath, virtualFiles, fileAccess),
|
|
531
616
|
async hashFileEntries(entries, hashAlgorithm) {
|
|
532
|
-
return addHashesToVirtualEntries(packagePath, virtualFiles, entries, hashAlgorithm);
|
|
617
|
+
return addHashesToVirtualEntries(packagePath, virtualFiles, fileAccess, entries, hashAlgorithm);
|
|
533
618
|
},
|
|
534
619
|
};
|
|
535
620
|
}
|
|
536
621
|
|
|
537
|
-
async function parseGgufInput(ggufPath) {
|
|
622
|
+
async function parseGgufInput(ggufPath, fileAccess) {
|
|
538
623
|
const ggufStats = await getPathStats(ggufPath, 'GGUF file');
|
|
539
624
|
const fileSize = Number(ggufStats.size);
|
|
540
625
|
const ggufSource = {
|
|
@@ -546,7 +631,7 @@ async function parseGgufInput(ggufPath) {
|
|
|
546
631
|
size: fileSize,
|
|
547
632
|
},
|
|
548
633
|
async readRange(offset, length) {
|
|
549
|
-
return readRange(ggufPath, offset, length);
|
|
634
|
+
return fileAccess.readRange(ggufPath, offset, length);
|
|
550
635
|
},
|
|
551
636
|
};
|
|
552
637
|
|
|
@@ -601,15 +686,15 @@ async function parseGgufInput(ggufPath) {
|
|
|
601
686
|
};
|
|
602
687
|
}
|
|
603
688
|
|
|
604
|
-
function buildNodeFileReaders() {
|
|
605
|
-
const readRangeFromFile = async (filePath, offset, length) => readRange(filePath, offset, length);
|
|
689
|
+
function buildNodeFileReaders(fileAccess) {
|
|
690
|
+
const readRangeFromFile = async (filePath, offset, length) => fileAccess.readRange(filePath, offset, length);
|
|
606
691
|
const streamRange = async function* (filePath, offset, length, options = {}) {
|
|
607
692
|
if (!Number.isFinite(offset) || !Number.isFinite(length) || length <= 0) {
|
|
608
693
|
return;
|
|
609
694
|
}
|
|
610
|
-
const
|
|
695
|
+
const fileSize = await fileAccess.getSize(filePath);
|
|
611
696
|
const start = Math.max(0, Math.floor(offset));
|
|
612
|
-
const end = Math.min(
|
|
697
|
+
const end = Math.min(fileSize, start + Math.floor(length));
|
|
613
698
|
if (end <= start) {
|
|
614
699
|
return;
|
|
615
700
|
}
|
|
@@ -645,6 +730,7 @@ function buildNodeFileReaders() {
|
|
|
645
730
|
streamRange,
|
|
646
731
|
readText,
|
|
647
732
|
readBinary,
|
|
733
|
+
close: fileAccess.close,
|
|
648
734
|
};
|
|
649
735
|
}
|
|
650
736
|
|
|
@@ -800,138 +886,159 @@ export async function resolveNodeSourceRuntimeBundle(options = {}) {
|
|
|
800
886
|
const verifyHashes = options.verifyHashes === true;
|
|
801
887
|
const resolvedInputPath = path.resolve(inputPath);
|
|
802
888
|
const stats = await getPathStats(resolvedInputPath, 'inputPath');
|
|
889
|
+
const fileAccess = createNodeFileAccess();
|
|
803
890
|
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
if (
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
parsed
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
if (!parsed && !isGgufPath(resolvedInputPath)) {
|
|
816
|
-
return null;
|
|
817
|
-
}
|
|
818
|
-
if (!parsed) {
|
|
819
|
-
parsed = await parseGgufInput(resolvedInputPath);
|
|
820
|
-
}
|
|
821
|
-
} else if (stats.isDirectory()) {
|
|
822
|
-
if (await fileExists(path.join(resolvedInputPath, 'manifest.json'))) {
|
|
823
|
-
return null;
|
|
824
|
-
}
|
|
825
|
-
parsed = await parseSafetensorsInput(resolvedInputPath);
|
|
826
|
-
if (!parsed) {
|
|
827
|
-
const entries = await fs.readdir(resolvedInputPath, { withFileTypes: true });
|
|
828
|
-
const ggufFiles = entries
|
|
829
|
-
.filter((entry) => entry.isFile() && isGgufPath(entry.name))
|
|
830
|
-
.map((entry) => entry.name)
|
|
831
|
-
.sort((left, right) => left.localeCompare(right));
|
|
832
|
-
if (ggufFiles.length === 1) {
|
|
833
|
-
parsed = await parseGgufInput(path.join(resolvedInputPath, ggufFiles[0]));
|
|
834
|
-
} else if (ggufFiles.length > 1) {
|
|
835
|
-
throw new Error(
|
|
836
|
-
`node source runtime: multiple GGUF files found in "${resolvedInputPath}": ${ggufFiles.join(', ')}.`
|
|
891
|
+
try {
|
|
892
|
+
let parsed = null;
|
|
893
|
+
if (stats.isFile()) {
|
|
894
|
+
if (isTflitePath(resolvedInputPath)) {
|
|
895
|
+
parsed = await parseTfliteInput(resolvedInputPath, fileAccess);
|
|
896
|
+
}
|
|
897
|
+
if (!parsed && isLiteRTTaskPath(resolvedInputPath)) {
|
|
898
|
+
parsed = await parseLiteRTPackageInput(
|
|
899
|
+
resolvedInputPath,
|
|
900
|
+
LITERT_PACKAGE_SOURCE_KIND_TASK,
|
|
901
|
+
fileAccess
|
|
837
902
|
);
|
|
838
903
|
}
|
|
839
|
-
if (!parsed) {
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
);
|
|
850
|
-
}
|
|
904
|
+
if (!parsed && isLiteRTLMPath(resolvedInputPath)) {
|
|
905
|
+
parsed = await parseLiteRTPackageInput(
|
|
906
|
+
resolvedInputPath,
|
|
907
|
+
LITERT_PACKAGE_SOURCE_KIND_LITERTLM,
|
|
908
|
+
fileAccess
|
|
909
|
+
);
|
|
910
|
+
}
|
|
911
|
+
if (!parsed && !isGgufPath(resolvedInputPath)) {
|
|
912
|
+
await fileAccess.close();
|
|
913
|
+
return null;
|
|
851
914
|
}
|
|
852
915
|
if (!parsed) {
|
|
853
|
-
|
|
854
|
-
.filter((entry) => entry.isFile() && isLiteRTTaskPath(entry.name))
|
|
855
|
-
.map((entry) => entry.name)
|
|
856
|
-
.sort((left, right) => left.localeCompare(right));
|
|
857
|
-
if (taskFiles.length === 1) {
|
|
858
|
-
parsed = await parseLiteRTPackageInput(
|
|
859
|
-
path.join(resolvedInputPath, taskFiles[0]),
|
|
860
|
-
LITERT_PACKAGE_SOURCE_KIND_TASK
|
|
861
|
-
);
|
|
862
|
-
} else if (taskFiles.length > 1) {
|
|
863
|
-
throw new Error(
|
|
864
|
-
`node source runtime: multiple LiteRT task files found in "${resolvedInputPath}": ${taskFiles.join(', ')}.`
|
|
865
|
-
);
|
|
866
|
-
}
|
|
916
|
+
parsed = await parseGgufInput(resolvedInputPath, fileAccess);
|
|
867
917
|
}
|
|
918
|
+
} else if (stats.isDirectory()) {
|
|
919
|
+
if (await fileExists(path.join(resolvedInputPath, 'manifest.json'))) {
|
|
920
|
+
await fileAccess.close();
|
|
921
|
+
return null;
|
|
922
|
+
}
|
|
923
|
+
parsed = await parseSafetensorsInput(resolvedInputPath, fileAccess);
|
|
868
924
|
if (!parsed) {
|
|
869
|
-
const
|
|
870
|
-
|
|
925
|
+
const entries = await fs.readdir(resolvedInputPath, { withFileTypes: true });
|
|
926
|
+
const ggufFiles = entries
|
|
927
|
+
.filter((entry) => entry.isFile() && isGgufPath(entry.name))
|
|
871
928
|
.map((entry) => entry.name)
|
|
872
929
|
.sort((left, right) => left.localeCompare(right));
|
|
873
|
-
if (
|
|
874
|
-
parsed = await
|
|
875
|
-
|
|
876
|
-
LITERT_PACKAGE_SOURCE_KIND_LITERTLM
|
|
877
|
-
);
|
|
878
|
-
} else if (litertLmFiles.length > 1) {
|
|
930
|
+
if (ggufFiles.length === 1) {
|
|
931
|
+
parsed = await parseGgufInput(path.join(resolvedInputPath, ggufFiles[0]), fileAccess);
|
|
932
|
+
} else if (ggufFiles.length > 1) {
|
|
879
933
|
throw new Error(
|
|
880
|
-
`node source runtime: multiple
|
|
934
|
+
`node source runtime: multiple GGUF files found in "${resolvedInputPath}": ${ggufFiles.join(', ')}.`
|
|
881
935
|
);
|
|
882
936
|
}
|
|
937
|
+
if (!parsed) {
|
|
938
|
+
const tfliteFiles = entries
|
|
939
|
+
.filter((entry) => entry.isFile() && isTflitePath(entry.name))
|
|
940
|
+
.map((entry) => entry.name)
|
|
941
|
+
.sort((left, right) => left.localeCompare(right));
|
|
942
|
+
if (tfliteFiles.length === 1) {
|
|
943
|
+
parsed = await parseTfliteInput(path.join(resolvedInputPath, tfliteFiles[0]), fileAccess);
|
|
944
|
+
} else if (tfliteFiles.length > 1) {
|
|
945
|
+
throw new Error(
|
|
946
|
+
`node source runtime: multiple TFLite files found in "${resolvedInputPath}": ${tfliteFiles.join(', ')}.`
|
|
947
|
+
);
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
if (!parsed) {
|
|
951
|
+
const taskFiles = entries
|
|
952
|
+
.filter((entry) => entry.isFile() && isLiteRTTaskPath(entry.name))
|
|
953
|
+
.map((entry) => entry.name)
|
|
954
|
+
.sort((left, right) => left.localeCompare(right));
|
|
955
|
+
if (taskFiles.length === 1) {
|
|
956
|
+
parsed = await parseLiteRTPackageInput(
|
|
957
|
+
path.join(resolvedInputPath, taskFiles[0]),
|
|
958
|
+
LITERT_PACKAGE_SOURCE_KIND_TASK,
|
|
959
|
+
fileAccess
|
|
960
|
+
);
|
|
961
|
+
} else if (taskFiles.length > 1) {
|
|
962
|
+
throw new Error(
|
|
963
|
+
`node source runtime: multiple LiteRT task files found in "${resolvedInputPath}": ${taskFiles.join(', ')}.`
|
|
964
|
+
);
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
if (!parsed) {
|
|
968
|
+
const litertLmFiles = entries
|
|
969
|
+
.filter((entry) => entry.isFile() && isLiteRTLMPath(entry.name))
|
|
970
|
+
.map((entry) => entry.name)
|
|
971
|
+
.sort((left, right) => left.localeCompare(right));
|
|
972
|
+
if (litertLmFiles.length === 1) {
|
|
973
|
+
parsed = await parseLiteRTPackageInput(
|
|
974
|
+
path.join(resolvedInputPath, litertLmFiles[0]),
|
|
975
|
+
LITERT_PACKAGE_SOURCE_KIND_LITERTLM,
|
|
976
|
+
fileAccess
|
|
977
|
+
);
|
|
978
|
+
} else if (litertLmFiles.length > 1) {
|
|
979
|
+
throw new Error(
|
|
980
|
+
`node source runtime: multiple LiteRT-LM files found in "${resolvedInputPath}": ${litertLmFiles.join(', ')}.`
|
|
981
|
+
);
|
|
982
|
+
}
|
|
983
|
+
}
|
|
883
984
|
}
|
|
985
|
+
} else {
|
|
986
|
+
await fileAccess.close();
|
|
987
|
+
return null;
|
|
884
988
|
}
|
|
885
|
-
} else {
|
|
886
|
-
return null;
|
|
887
|
-
}
|
|
888
989
|
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
990
|
+
if (!parsed) {
|
|
991
|
+
await fileAccess.close();
|
|
992
|
+
return null;
|
|
993
|
+
}
|
|
892
994
|
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
995
|
+
const loadingConfig = resolveLoadingConfig(options.runtimeConfig ?? null);
|
|
996
|
+
const resolvedMemoryBudgetBytes = resolveResidentBudgetBytes(loadingConfig);
|
|
997
|
+
assertSourceRuntimeFitsResidentBudget(parsed, loadingConfig, resolvedMemoryBudgetBytes);
|
|
998
|
+
const {
|
|
999
|
+
model,
|
|
1000
|
+
shardSources,
|
|
1001
|
+
sourceKind,
|
|
1002
|
+
} = await resolveSourceRuntimeBundleFromParsedArtifact({
|
|
1003
|
+
parsedArtifact: parsed,
|
|
1004
|
+
requestedModelId: options.modelId || null,
|
|
1005
|
+
runtimeLabel: 'node source runtime',
|
|
1006
|
+
logCategory: 'NodeSourceRuntime',
|
|
1007
|
+
hashFileEntries: typeof parsed?.hashFileEntries === 'function'
|
|
1008
|
+
? (entries, hashAlgorithm) => parsed.hashFileEntries(entries, hashAlgorithm)
|
|
1009
|
+
: addHashesToFileEntries,
|
|
1010
|
+
});
|
|
909
1011
|
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
1012
|
+
const readers = parsed?.storageReaders ?? buildNodeFileReaders(fileAccess);
|
|
1013
|
+
const storageContext = createSourceStorageContext({
|
|
1014
|
+
model,
|
|
1015
|
+
shardSources,
|
|
1016
|
+
readRange: readers.readRange,
|
|
1017
|
+
streamRange: readers.streamRange,
|
|
1018
|
+
readText: readers.readText,
|
|
1019
|
+
readBinary: readers.readBinary,
|
|
1020
|
+
close: readers.close,
|
|
1021
|
+
tokenizerJsonPath: parsed.tokenizerJsonPath,
|
|
1022
|
+
tokenizerModelPath: parsed.tokenizerModelPath,
|
|
1023
|
+
verifyHashes,
|
|
1024
|
+
sourceHashesTrusted: true,
|
|
1025
|
+
});
|
|
923
1026
|
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
1027
|
+
log.info(
|
|
1028
|
+
'NodeSourceRuntime',
|
|
1029
|
+
`Source runtime ready: ${model.modelId} (${sourceKind}, ${parsed.tensors.length} tensors)`
|
|
1030
|
+
);
|
|
928
1031
|
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
1032
|
+
return {
|
|
1033
|
+
model,
|
|
1034
|
+
manifest: model,
|
|
1035
|
+
storageContext,
|
|
1036
|
+
sourceKind,
|
|
1037
|
+
sourceRoot: parsed.sourceRoot,
|
|
1038
|
+
resolvedMemoryBudgetBytes,
|
|
1039
|
+
};
|
|
1040
|
+
} catch (error) {
|
|
1041
|
+
await fileAccess.close();
|
|
1042
|
+
throw error;
|
|
1043
|
+
}
|
|
937
1044
|
}
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
import { downloadModel, estimateTimeRemaining, formatSpeed } from '../storage/downloader.js';
|
|
9
9
|
import { isOPFSAvailable, formatBytes } from '../storage/quota.js';
|
|
10
10
|
import { parseManifest, getManifestUrl } from '../formats/rdrr/index.js';
|
|
11
|
+
import { cloneJsonValue } from '../utils/clone-json.js';
|
|
11
12
|
import { log } from '../debug/index.js';
|
|
12
13
|
import {
|
|
13
14
|
resolveSourceArtifact,
|
|
@@ -47,10 +48,6 @@ function hasSameShardSet(aManifest, bManifest) {
|
|
|
47
48
|
return true;
|
|
48
49
|
}
|
|
49
50
|
|
|
50
|
-
function cloneJsonValue(value) {
|
|
51
|
-
return value == null ? value : JSON.parse(JSON.stringify(value));
|
|
52
|
-
}
|
|
53
|
-
|
|
54
51
|
function preserveCachedSourceRuntimeMetadata(remoteManifest, cachedManifest) {
|
|
55
52
|
const cachedSourceRuntime = cachedManifest?.metadata?.sourceRuntime;
|
|
56
53
|
if (!cachedSourceRuntime || typeof cachedSourceRuntime !== 'object') {
|