doppler-gpu 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +19 -15
- package/package.json +77 -29
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +64 -6
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +524 -350
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +12 -2
- package/src/config/schema/doppler.schema.js +108 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +68 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +76 -0
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +107 -0
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +3 -21
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +6 -5
- package/src/gpu/kernels/index.js +3 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +12 -0
- package/src/inference/browser-harness-text-helpers.js +223 -14
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +264 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +22 -27
- package/src/inference/pipelines/text/embed.js +0 -39
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +4 -5
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +8 -3
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +223 -547
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +5 -2
- package/src/inference/pipelines/text/init.js +183 -724
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.js +2 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +120 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +223 -27
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +13 -3
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +11 -3
- package/src/storage/artifact-storage-context.js +231 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +21 -24
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +2 -0
- package/src/tooling/source-runtime-bundle.js +3 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
parseLiteRTTaskFromSource,
|
|
13
13
|
} from '../formats/litert/types.js';
|
|
14
14
|
import { resolveDirectSourcePackageProfile } from './source-package-profiles.js';
|
|
15
|
+
import { cloneJsonValue } from '../utils/clone-json.js';
|
|
15
16
|
|
|
16
17
|
export const LITERT_PACKAGE_SOURCE_KIND_TASK = 'litert-task';
|
|
17
18
|
export const LITERT_PACKAGE_SOURCE_KIND_LITERTLM = 'litertlm';
|
|
@@ -20,13 +21,6 @@ function normalizeText(value) {
|
|
|
20
21
|
return String(value || '').trim();
|
|
21
22
|
}
|
|
22
23
|
|
|
23
|
-
function cloneJsonValue(value) {
|
|
24
|
-
if (typeof structuredClone === 'function') {
|
|
25
|
-
return structuredClone(value);
|
|
26
|
-
}
|
|
27
|
-
return JSON.parse(JSON.stringify(value));
|
|
28
|
-
}
|
|
29
|
-
|
|
30
24
|
function createVirtualFile(path, offset, size, kind, options = {}) {
|
|
31
25
|
return {
|
|
32
26
|
path,
|
|
@@ -501,19 +495,23 @@ function createLiteRTAxisTensor(
|
|
|
501
495
|
const scaleTensorSourceTransform = scaleTensor.sourceTransform;
|
|
502
496
|
const scaleCompanionDtype = String(scaleTensor.sourceDtype || '').toUpperCase();
|
|
503
497
|
const hasUint8ScaleCompanion = scaleCompanionDtype === 'UINT8';
|
|
504
|
-
|
|
498
|
+
|
|
499
|
+
// When the scale companion is UINT8 without affine_dequant metadata, the
|
|
500
|
+
// UINT8 container holds packed F32 row-scales (4 bytes per F32 value).
|
|
501
|
+
// This is the MediaPipe symmetric quantization convention used by LiteRT-LM
|
|
502
|
+
// .task weight bags (e.g. Gemma 4 E2B per_layer_embeddings): weight bytes
|
|
503
|
+
// are semantically INT8 (signed, zero_point=0), scale = max(|row|) / 127.
|
|
504
|
+
// The companion bytes are native F32 — no affine dequant needed.
|
|
505
|
+
const isPackedF32ScaleCompanion = hasUint8ScaleCompanion
|
|
506
|
+
&& (!scaleTensorSourceTransform || scaleTensorSourceTransform.kind !== 'affine_dequant');
|
|
507
|
+
|
|
508
|
+
const scaleCompanionDequant = hasUint8ScaleCompanion && !isPackedF32ScaleCompanion
|
|
505
509
|
? {
|
|
506
510
|
scale: Number(scaleTensorSourceTransform?.scale),
|
|
507
511
|
zeroPoint: Number(scaleTensorSourceTransform?.zeroPoint),
|
|
508
512
|
}
|
|
509
513
|
: null;
|
|
510
|
-
if (hasUint8ScaleCompanion) {
|
|
511
|
-
if (!scaleTensorSourceTransform || scaleTensorSourceTransform.kind !== 'affine_dequant') {
|
|
512
|
-
throw new Error(
|
|
513
|
-
`direct-source runtime: LiteRT tensor "${rawTensor.name}" has UINT8 scale companion "${rawTensor.name}_quantized_scale" ` +
|
|
514
|
-
'without affine_dequant metadata.'
|
|
515
|
-
);
|
|
516
|
-
}
|
|
514
|
+
if (hasUint8ScaleCompanion && !isPackedF32ScaleCompanion) {
|
|
517
515
|
if (!Number.isFinite(scaleCompanionDequant.scale) || scaleCompanionDequant.scale <= 0) {
|
|
518
516
|
throw new Error(
|
|
519
517
|
`direct-source runtime: LiteRT tensor "${rawTensor.name}" has invalid scale companion affine_dequant scale ${scaleTensorSourceTransform.scale}.`
|
|
@@ -591,7 +589,9 @@ function createLiteRTAxisTensor(
|
|
|
591
589
|
sourceTransform: {
|
|
592
590
|
kind: 'litert_axis_dequant',
|
|
593
591
|
scheme: 'per_axis_affine',
|
|
594
|
-
sourceDtype: layout.sourceDtype
|
|
592
|
+
sourceDtype: isPackedF32ScaleCompanion && layout.sourceDtype === 'UINT8'
|
|
593
|
+
? 'INT8'
|
|
594
|
+
: layout.sourceDtype,
|
|
595
595
|
targetDtype: 'F16',
|
|
596
596
|
storageEncoding: layout.storageEncoding,
|
|
597
597
|
scaleSemantics: scaleContract.scaleSemantics,
|
|
@@ -601,7 +601,7 @@ function createLiteRTAxisTensor(
|
|
|
601
601
|
scaleSourcePath: sourcePath,
|
|
602
602
|
scaleOffset: scaleTensor.offset,
|
|
603
603
|
scaleSize: scaleTensor.size,
|
|
604
|
-
...(hasUint8ScaleCompanion
|
|
604
|
+
...(hasUint8ScaleCompanion && !isPackedF32ScaleCompanion
|
|
605
605
|
? {
|
|
606
606
|
scaleCompanionDtype,
|
|
607
607
|
scaleCompanionDequant,
|
|
@@ -831,11 +831,9 @@ function normalizeGemma4LiteRTTensors(parsedTFLite, sourcePath, runtimeProfile)
|
|
|
831
831
|
resolvedLogicalShape,
|
|
832
832
|
resolvedStorageShape,
|
|
833
833
|
resolvedQuantAxis,
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
}
|
|
838
|
-
: null
|
|
834
|
+
{
|
|
835
|
+
scaleSemantics: options.scaleSemantics || 'step',
|
|
836
|
+
}
|
|
839
837
|
)
|
|
840
838
|
);
|
|
841
839
|
};
|
|
@@ -855,8 +853,7 @@ function normalizeGemma4LiteRTTensors(parsedTFLite, sourcePath, runtimeProfile)
|
|
|
855
853
|
4,
|
|
856
854
|
[0, 1, 2, 3],
|
|
857
855
|
{
|
|
858
|
-
scaleSemantics: '
|
|
859
|
-
scaleDivisor: 3,
|
|
856
|
+
scaleSemantics: 'step',
|
|
860
857
|
}
|
|
861
858
|
)
|
|
862
859
|
);
|
|
@@ -42,6 +42,21 @@ export declare function runBrowserCommandInNode(
|
|
|
42
42
|
options?: NodeBrowserCommandRunOptions
|
|
43
43
|
): Promise<BrowserCommandRunResult>;
|
|
44
44
|
|
|
45
|
+
export declare function runBrowserCommandEvaluationWithTimeout<T>(
|
|
46
|
+
operation: () => Promise<T>,
|
|
47
|
+
timeoutMs: number
|
|
48
|
+
): Promise<T>;
|
|
49
|
+
|
|
45
50
|
export declare function normalizeNodeBrowserCommand(
|
|
46
51
|
commandRequest: ToolingCommandRequestInput
|
|
47
52
|
): ToolingCommandRequest;
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Validate a browser-relay response envelope and stamp the original
|
|
56
|
+
* request onto it. Throws when the envelope does not match the success
|
|
57
|
+
* shape.
|
|
58
|
+
*/
|
|
59
|
+
export declare function finalizeBrowserRelayResponse(
|
|
60
|
+
response: unknown,
|
|
61
|
+
request: ToolingCommandRequest
|
|
62
|
+
): BrowserCommandRunResult & { request: ToolingCommandRequest };
|
|
@@ -475,10 +475,10 @@ function normalizeBrowserArgs(value) {
|
|
|
475
475
|
|
|
476
476
|
return value.map((arg) => {
|
|
477
477
|
if (arg === undefined || arg === null) {
|
|
478
|
-
throw new Error('browser command:
|
|
478
|
+
throw new Error('browser command: browserArgs values must be strings.');
|
|
479
479
|
}
|
|
480
480
|
if (typeof arg !== 'string') {
|
|
481
|
-
throw new Error('browser command:
|
|
481
|
+
throw new Error('browser command: browserArgs values must be strings.');
|
|
482
482
|
}
|
|
483
483
|
return arg.trim();
|
|
484
484
|
}).filter((arg) => arg.length > 0);
|
|
@@ -505,6 +505,7 @@ const PLATFORM_WEBGPU_ARGS = Object.freeze({
|
|
|
505
505
|
]),
|
|
506
506
|
win32: Object.freeze([]),
|
|
507
507
|
});
|
|
508
|
+
const BROWSER_LAUNCH_HINT = 'Install Playwright browsers (npx playwright install) or set run.browser.channel / run.browser.executablePath.';
|
|
508
509
|
|
|
509
510
|
function uniqueArgs(args) {
|
|
510
511
|
return [...new Set(args)];
|
|
@@ -588,6 +589,57 @@ function hasCrashRecoveryArgs(args = []) {
|
|
|
588
589
|
return CRASH_RECOVERY_BROWSER_ARGS.every((arg) => argSet.has(arg));
|
|
589
590
|
}
|
|
590
591
|
|
|
592
|
+
function formatBrowserEvaluationError(payload) {
|
|
593
|
+
if (!payload || typeof payload !== 'object') {
|
|
594
|
+
return new Error('browser command runner failed with an unserializable error.');
|
|
595
|
+
}
|
|
596
|
+
const message = asNonEmptyString(payload.message) || 'Unknown browser command error';
|
|
597
|
+
const stack = asNonEmptyString(payload.stack);
|
|
598
|
+
const causeMessage = asNonEmptyString(payload.cause?.message);
|
|
599
|
+
const text = [
|
|
600
|
+
`browser command runner failed: ${message}`,
|
|
601
|
+
stack,
|
|
602
|
+
causeMessage ? `Caused by: ${causeMessage}` : null,
|
|
603
|
+
].filter(Boolean).join('\n');
|
|
604
|
+
const error = new Error(text);
|
|
605
|
+
error.name = asNonEmptyString(payload.name) || 'BrowserCommandError';
|
|
606
|
+
error.code = asNonEmptyString(payload.code) || null;
|
|
607
|
+
error.retryable = typeof payload.retryable === 'boolean' ? payload.retryable : null;
|
|
608
|
+
error.details = {
|
|
609
|
+
...(payload.details && typeof payload.details === 'object' ? payload.details : {}),
|
|
610
|
+
browserErrorName: asNonEmptyString(payload.name),
|
|
611
|
+
browserStack: stack,
|
|
612
|
+
browserCause: payload.cause && typeof payload.cause === 'object'
|
|
613
|
+
? {
|
|
614
|
+
name: asNonEmptyString(payload.cause.name),
|
|
615
|
+
message: asNonEmptyString(payload.cause.message),
|
|
616
|
+
stack: asNonEmptyString(payload.cause.stack),
|
|
617
|
+
code: asNonEmptyString(payload.cause.code),
|
|
618
|
+
}
|
|
619
|
+
: null,
|
|
620
|
+
};
|
|
621
|
+
return error;
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
export async function runBrowserCommandEvaluationWithTimeout(operation, timeoutMs) {
|
|
625
|
+
let timeoutId = null;
|
|
626
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
627
|
+
timeoutId = setTimeout(() => {
|
|
628
|
+
reject(new Error(`browser command: runner did not finish within ${timeoutMs}ms.`));
|
|
629
|
+
}, timeoutMs);
|
|
630
|
+
});
|
|
631
|
+
try {
|
|
632
|
+
return await Promise.race([
|
|
633
|
+
operation(),
|
|
634
|
+
timeoutPromise,
|
|
635
|
+
]);
|
|
636
|
+
} finally {
|
|
637
|
+
if (timeoutId !== null) {
|
|
638
|
+
clearTimeout(timeoutId);
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
|
|
591
643
|
function browserLaunchArgs(extraArgs = []) {
|
|
592
644
|
const platformArgs = PLATFORM_WEBGPU_ARGS[process.platform] ?? [];
|
|
593
645
|
return uniqueArgs([...DEFAULT_WEBGPU_BROWSER_ARGS, ...platformArgs, ...extraArgs]);
|
|
@@ -606,7 +658,7 @@ async function launchBrowser(chromium, launchOptions, options = {}) {
|
|
|
606
658
|
} catch (error) {
|
|
607
659
|
const message = error?.message || String(error);
|
|
608
660
|
throw new Error(
|
|
609
|
-
`browser command: failed to launch browser (${message}).
|
|
661
|
+
`browser command: failed to launch browser (${message}). ${BROWSER_LAUNCH_HINT}`
|
|
610
662
|
);
|
|
611
663
|
}
|
|
612
664
|
}
|
|
@@ -632,7 +684,7 @@ async function launchBrowser(chromium, launchOptions, options = {}) {
|
|
|
632
684
|
throw new Error(
|
|
633
685
|
`browser command: failed to launch browser (${allErrors}). ` +
|
|
634
686
|
`Tried default channels: ${resolveDefaultChannels().join(', ')}. ` +
|
|
635
|
-
|
|
687
|
+
BROWSER_LAUNCH_HINT
|
|
636
688
|
);
|
|
637
689
|
}
|
|
638
690
|
};
|
|
@@ -666,14 +718,14 @@ async function launchBrowser(chromium, launchOptions, options = {}) {
|
|
|
666
718
|
if (isRecoverablePersistentLaunchError(retryMessage)) {
|
|
667
719
|
throw new Error(
|
|
668
720
|
`browser command: failed to launch browser with crash recovery enabled (${retryMessage}). ` +
|
|
669
|
-
|
|
721
|
+
BROWSER_LAUNCH_HINT
|
|
670
722
|
);
|
|
671
723
|
}
|
|
672
724
|
|
|
673
725
|
throw new Error(
|
|
674
726
|
`browser command: failed to launch browser (${retryMessage}). ` +
|
|
675
727
|
`Tried default channels: ${resolveDefaultChannels().join(', ')}. ` +
|
|
676
|
-
|
|
728
|
+
BROWSER_LAUNCH_HINT
|
|
677
729
|
);
|
|
678
730
|
}
|
|
679
731
|
|
|
@@ -692,7 +744,7 @@ async function launchPersistentBrowser(chromium, userDataDir, launchOptions, opt
|
|
|
692
744
|
} catch (error) {
|
|
693
745
|
const message = error?.message || String(error);
|
|
694
746
|
throw new Error(
|
|
695
|
-
`browser command: failed to launch persistent browser (${message}).
|
|
747
|
+
`browser command: failed to launch persistent browser (${message}). ${BROWSER_LAUNCH_HINT}`
|
|
696
748
|
);
|
|
697
749
|
}
|
|
698
750
|
}
|
|
@@ -734,14 +786,14 @@ async function launchPersistentBrowser(chromium, userDataDir, launchOptions, opt
|
|
|
734
786
|
throw new Error(
|
|
735
787
|
`browser command: failed to launch persistent browser with crash recovery enabled (${retryMessage}). ` +
|
|
736
788
|
`Tried default channels: ${resolveDefaultChannels().join(', ')}. ` +
|
|
737
|
-
|
|
789
|
+
BROWSER_LAUNCH_HINT
|
|
738
790
|
);
|
|
739
791
|
}
|
|
740
792
|
|
|
741
793
|
throw new Error(
|
|
742
794
|
`browser command: failed to launch persistent browser (${retryMessage}). ` +
|
|
743
795
|
`Tried default channels: ${resolveDefaultChannels().join(', ')}. ` +
|
|
744
|
-
|
|
796
|
+
BROWSER_LAUNCH_HINT
|
|
745
797
|
);
|
|
746
798
|
}
|
|
747
799
|
|
|
@@ -791,7 +843,7 @@ export async function runBrowserCommandInNode(commandRequest, options = {}) {
|
|
|
791
843
|
}).catch((error) => {
|
|
792
844
|
const message = error?.message || String(error);
|
|
793
845
|
throw new Error(
|
|
794
|
-
`browser command: failed to start static server (${message}).
|
|
846
|
+
`browser command: failed to start static server (${message}). Set run.browser.baseUrl to reuse an existing server.`
|
|
795
847
|
);
|
|
796
848
|
});
|
|
797
849
|
|
|
@@ -945,17 +997,49 @@ export async function runBrowserCommandInNode(commandRequest, options = {}) {
|
|
|
945
997
|
}
|
|
946
998
|
}
|
|
947
999
|
|
|
948
|
-
const response = await page.evaluate(async (payload) => {
|
|
1000
|
+
const response = await runBrowserCommandEvaluationWithTimeout(() => page.evaluate(async (payload) => {
|
|
1001
|
+
const serializeError = (error, depth = 0) => {
|
|
1002
|
+
if (!error || typeof error !== 'object') {
|
|
1003
|
+
return {
|
|
1004
|
+
name: null,
|
|
1005
|
+
message: String(error || 'Unknown browser error'),
|
|
1006
|
+
stack: null,
|
|
1007
|
+
code: null,
|
|
1008
|
+
details: null,
|
|
1009
|
+
retryable: null,
|
|
1010
|
+
cause: null,
|
|
1011
|
+
};
|
|
1012
|
+
}
|
|
1013
|
+
return {
|
|
1014
|
+
name: typeof error.name === 'string' ? error.name : null,
|
|
1015
|
+
message: typeof error.message === 'string' ? error.message : String(error),
|
|
1016
|
+
stack: typeof error.stack === 'string' ? error.stack : null,
|
|
1017
|
+
code: typeof error.code === 'string' ? error.code : null,
|
|
1018
|
+
details: error.details && typeof error.details === 'object' ? error.details : null,
|
|
1019
|
+
retryable: typeof error.retryable === 'boolean' ? error.retryable : null,
|
|
1020
|
+
cause: depth < 2 ? serializeError(error.cause, depth + 1) : null,
|
|
1021
|
+
};
|
|
1022
|
+
};
|
|
949
1023
|
if (typeof globalThis.__dopplerRunBrowserCommand !== 'function') {
|
|
950
1024
|
throw new Error('browser command runner is missing globalThis.__dopplerRunBrowserCommand');
|
|
951
1025
|
}
|
|
952
|
-
|
|
1026
|
+
try {
|
|
1027
|
+
return await globalThis.__dopplerRunBrowserCommand(payload.request, payload.options || {});
|
|
1028
|
+
} catch (error) {
|
|
1029
|
+
return {
|
|
1030
|
+
__dopplerBrowserError: serializeError(error),
|
|
1031
|
+
};
|
|
1032
|
+
}
|
|
953
1033
|
}, {
|
|
954
1034
|
request: relayRequest,
|
|
955
1035
|
options: {
|
|
956
1036
|
runtimeLoadOptions: options.runtimeLoadOptions || {},
|
|
957
1037
|
},
|
|
958
|
-
});
|
|
1038
|
+
}), timeoutMs);
|
|
1039
|
+
|
|
1040
|
+
if (response?.__dopplerBrowserError) {
|
|
1041
|
+
throw formatBrowserEvaluationError(response.__dopplerBrowserError);
|
|
1042
|
+
}
|
|
959
1043
|
|
|
960
1044
|
return finalizeBrowserRelayResponse(response, effectiveRequest);
|
|
961
1045
|
} catch (error) {
|
|
@@ -11,8 +11,10 @@ import { convertSafetensorsDirectory } from './node-converter.js';
|
|
|
11
11
|
import { installNodeFileFetchShim } from './node-file-fetch.js';
|
|
12
12
|
import { bootstrapNodeWebGPU } from './node-webgpu.js';
|
|
13
13
|
import { runDiagnoseCommand } from './diagnose-runner.js';
|
|
14
|
+
import { checkProgramBundleParity } from './program-bundle-parity.js';
|
|
14
15
|
import { applyRuntimeInputs, buildSuiteOptions } from './command-runner-shared.js';
|
|
15
16
|
import { runWithRuntimeIsolation } from './command-runner-shared.js';
|
|
17
|
+
import { refreshManifestIntegrity } from './rdrr-integrity-refresh.js';
|
|
16
18
|
import { isPlainObject } from '../utils/plain-object.js';
|
|
17
19
|
import {
|
|
18
20
|
getActiveKernelPath,
|
|
@@ -125,6 +127,7 @@ export async function runNodeCommand(commandRequest, options = {}) {
|
|
|
125
127
|
outputDir: request.outputDir,
|
|
126
128
|
converterConfig,
|
|
127
129
|
execution,
|
|
130
|
+
configPath: convertPayload?.configPath ?? null,
|
|
128
131
|
onProgress: validatedOptions.onProgress,
|
|
129
132
|
});
|
|
130
133
|
return createToolingSuccessEnvelope({
|
|
@@ -134,6 +137,21 @@ export async function runNodeCommand(commandRequest, options = {}) {
|
|
|
134
137
|
});
|
|
135
138
|
}
|
|
136
139
|
|
|
140
|
+
if (request.command === 'refresh-integrity') {
|
|
141
|
+
const result = await refreshManifestIntegrity({
|
|
142
|
+
modelDir: request.modelDir,
|
|
143
|
+
manifestPath: request.manifestPath,
|
|
144
|
+
blockSize: request.blockSize ?? undefined,
|
|
145
|
+
dryRun: request.dryRun === true,
|
|
146
|
+
skipShardCheck: request.skipShardCheck === true,
|
|
147
|
+
});
|
|
148
|
+
return createToolingSuccessEnvelope({
|
|
149
|
+
surface: 'node',
|
|
150
|
+
request,
|
|
151
|
+
result,
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
|
|
137
155
|
if (request.command === 'lora' || request.command === 'distill') {
|
|
138
156
|
const gpuOptionalActions = new Set(['compare', 'quality-gate', 'subsets']);
|
|
139
157
|
installNodeFileFetchShim();
|
|
@@ -159,6 +177,24 @@ export async function runNodeCommand(commandRequest, options = {}) {
|
|
|
159
177
|
});
|
|
160
178
|
}
|
|
161
179
|
|
|
180
|
+
if (
|
|
181
|
+
request.command === 'verify'
|
|
182
|
+
&& request.workload === 'inference'
|
|
183
|
+
&& request.workloadType === 'program-bundle'
|
|
184
|
+
) {
|
|
185
|
+
const result = await checkProgramBundleParity({
|
|
186
|
+
bundle: request.programBundle ?? undefined,
|
|
187
|
+
bundlePath: request.programBundlePath ?? undefined,
|
|
188
|
+
providers: request.parityProviders ?? undefined,
|
|
189
|
+
mode: request.programBundleParityMode ?? 'contract',
|
|
190
|
+
});
|
|
191
|
+
return createToolingSuccessEnvelope({
|
|
192
|
+
surface: 'node',
|
|
193
|
+
request,
|
|
194
|
+
result,
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
|
|
162
198
|
await assertNodeWebGPUSupport();
|
|
163
199
|
const modules = await loadRuntimeModules();
|
|
164
200
|
const runtimeBridge = {
|
|
@@ -150,10 +150,22 @@ export class NodeConvertWorkerPool {
|
|
|
150
150
|
if (!(resultBytes instanceof ArrayBuffer)) {
|
|
151
151
|
job.reject(new Error('node convert worker returned invalid tensorData.'));
|
|
152
152
|
} else {
|
|
153
|
+
const companionBytes = message?.result?.companionData;
|
|
154
|
+
if (companionBytes != null && !(companionBytes instanceof ArrayBuffer)) {
|
|
155
|
+
job.reject(new Error('node convert worker returned invalid companionData.'));
|
|
156
|
+
this.#dispatch();
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
153
159
|
job.resolve({
|
|
154
160
|
tensorData: new Uint8Array(resultBytes),
|
|
155
161
|
outDtype: message?.result?.outDtype ?? null,
|
|
156
162
|
outLayout: message?.result?.outLayout ?? null,
|
|
163
|
+
...(companionBytes instanceof ArrayBuffer
|
|
164
|
+
? { companionData: new Uint8Array(companionBytes) }
|
|
165
|
+
: {}),
|
|
166
|
+
...(message?.result?.sourceTransform
|
|
167
|
+
? { sourceTransform: message.result.sourceTransform }
|
|
168
|
+
: {}),
|
|
157
169
|
});
|
|
158
170
|
}
|
|
159
171
|
this.#dispatch();
|
|
@@ -36,6 +36,13 @@ parentPort.on('message', (message) => {
|
|
|
36
36
|
|
|
37
37
|
const transformed = transformTensorBytes(tensor, new Uint8Array(rawData), job.transformContext ?? {});
|
|
38
38
|
const outBytes = toOwnedUint8Array(transformed.tensorData);
|
|
39
|
+
const companionBytes = transformed.companionData instanceof Uint8Array
|
|
40
|
+
? toOwnedUint8Array(transformed.companionData)
|
|
41
|
+
: null;
|
|
42
|
+
const transferList = [outBytes.buffer];
|
|
43
|
+
if (companionBytes && companionBytes.buffer !== outBytes.buffer) {
|
|
44
|
+
transferList.push(companionBytes.buffer);
|
|
45
|
+
}
|
|
39
46
|
parentPort.postMessage({
|
|
40
47
|
id,
|
|
41
48
|
ok: true,
|
|
@@ -43,8 +50,10 @@ parentPort.on('message', (message) => {
|
|
|
43
50
|
tensorData: outBytes.buffer,
|
|
44
51
|
outDtype: transformed.outDtype ?? tensor.dtype ?? null,
|
|
45
52
|
outLayout: transformed.outLayout ?? null,
|
|
53
|
+
...(companionBytes ? { companionData: companionBytes.buffer } : {}),
|
|
54
|
+
...(transformed.sourceTransform ? { sourceTransform: transformed.sourceTransform } : {}),
|
|
46
55
|
},
|
|
47
|
-
},
|
|
56
|
+
}, transferList);
|
|
48
57
|
} catch (error) {
|
|
49
58
|
const messageText = error instanceof Error ? error.message : String(error);
|
|
50
59
|
const stack = error instanceof Error ? error.stack : null;
|
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
import { createReadStream } from 'node:fs';
|
|
1
2
|
import fs from 'node:fs/promises';
|
|
2
3
|
import os from 'node:os';
|
|
3
4
|
import path from 'node:path';
|
|
5
|
+
import { pathToFileURL } from 'node:url';
|
|
4
6
|
import { installNodeFileFetchShim } from './node-file-fetch.js';
|
|
5
7
|
import { NodeConvertWorkerPool } from './node-convert-worker-pool.js';
|
|
6
8
|
import { bootstrapNodeWebGPU } from './node-webgpu.js';
|
|
9
|
+
import { buildManifestIntegrityFromModelDir } from './rdrr-integrity-refresh.js';
|
|
7
10
|
import { isPlainObject } from '../utils/plain-object.js';
|
|
8
11
|
import { selectRuleValue } from '../rules/rule-registry.js';
|
|
9
12
|
import { log, trace } from '../debug/index.js';
|
|
@@ -159,10 +162,14 @@ function normalizeWorkerTransformResult(result, tensor) {
|
|
|
159
162
|
tensorData: result.tensorData,
|
|
160
163
|
outDtype: result.outDtype ?? tensor.dtype,
|
|
161
164
|
outLayout: result.outLayout ?? null,
|
|
165
|
+
...(result.companionData instanceof Uint8Array
|
|
166
|
+
? { companionData: result.companionData }
|
|
167
|
+
: {}),
|
|
168
|
+
...(result.sourceTransform ? { sourceTransform: result.sourceTransform } : {}),
|
|
162
169
|
};
|
|
163
170
|
}
|
|
164
171
|
|
|
165
|
-
const MAX_NODE_CONVERT_BUFFER_BYTES =
|
|
172
|
+
const MAX_NODE_CONVERT_BUFFER_BYTES = Math.min(Buffer.kMaxLength, 0x7fff_ffff);
|
|
166
173
|
|
|
167
174
|
function resolveRowChunkTransformPlan(input) {
|
|
168
175
|
const tensor = input?.tensor;
|
|
@@ -435,6 +442,13 @@ function readOptionalNonEmptyString(value) {
|
|
|
435
442
|
return trimmed || null;
|
|
436
443
|
}
|
|
437
444
|
|
|
445
|
+
function toRepoRelativePath(filePath) {
|
|
446
|
+
const normalized = readOptionalNonEmptyString(filePath);
|
|
447
|
+
if (!normalized) return null;
|
|
448
|
+
const relative = path.relative(process.cwd(), path.resolve(normalized)).replace(/\\/g, '/');
|
|
449
|
+
return relative && !relative.startsWith('..') ? relative : path.resolve(normalized);
|
|
450
|
+
}
|
|
451
|
+
|
|
438
452
|
function resolveConfiguredModelId(explicitModelId, converterConfig) {
|
|
439
453
|
return (
|
|
440
454
|
readOptionalNonEmptyString(explicitModelId)
|
|
@@ -533,6 +547,7 @@ async function resolveGgufPathFromDirectory(inputDir) {
|
|
|
533
547
|
|
|
534
548
|
function createFileRangeReader() {
|
|
535
549
|
const handleMap = new Map();
|
|
550
|
+
const maxNodeFdReadInt32 = 0x7fff_ffff;
|
|
536
551
|
|
|
537
552
|
async function getHandleEntry(filePath) {
|
|
538
553
|
const existingPromise = handleMap.get(filePath);
|
|
@@ -576,6 +591,25 @@ function createFileRangeReader() {
|
|
|
576
591
|
return new ArrayBuffer(0);
|
|
577
592
|
}
|
|
578
593
|
|
|
594
|
+
if (start > maxNodeFdReadInt32 || (end - start) > maxNodeFdReadInt32) {
|
|
595
|
+
const chunks = [];
|
|
596
|
+
let totalBytes = 0;
|
|
597
|
+
await new Promise((resolve, reject) => {
|
|
598
|
+
const stream = createReadStream(filePath, {
|
|
599
|
+
start,
|
|
600
|
+
end: end - 1,
|
|
601
|
+
});
|
|
602
|
+
stream.on('data', (chunk) => {
|
|
603
|
+
chunks.push(chunk);
|
|
604
|
+
totalBytes += chunk.byteLength;
|
|
605
|
+
});
|
|
606
|
+
stream.on('end', resolve);
|
|
607
|
+
stream.on('error', reject);
|
|
608
|
+
});
|
|
609
|
+
const out = Buffer.concat(chunks, totalBytes);
|
|
610
|
+
return out.buffer.slice(out.byteOffset, out.byteOffset + out.byteLength);
|
|
611
|
+
}
|
|
612
|
+
|
|
579
613
|
const out = Buffer.allocUnsafe(end - start);
|
|
580
614
|
await entry.fd.read(out, 0, out.length, start);
|
|
581
615
|
return out.buffer.slice(out.byteOffset, out.byteOffset + out.byteLength);
|
|
@@ -660,6 +694,10 @@ function createNodeConvertIO(outputDir, options) {
|
|
|
660
694
|
async readTensorData(tensor) {
|
|
661
695
|
return readRange(tensor.sourcePath, tensor.offset, tensor.size);
|
|
662
696
|
},
|
|
697
|
+
async readShardRange(index, offset, length) {
|
|
698
|
+
const filename = generateShardFilename(index);
|
|
699
|
+
return readRange(path.join(outputDir, filename), offset, length);
|
|
700
|
+
},
|
|
663
701
|
async writeShard(index, data) {
|
|
664
702
|
const filename = generateShardFilename(index);
|
|
665
703
|
await fs.writeFile(path.join(outputDir, filename), data);
|
|
@@ -804,6 +842,7 @@ function createNodeTensorTransformer(options) {
|
|
|
804
842
|
const transformed = await pool.transformTensor(chunkTensor, chunkTensorData, {
|
|
805
843
|
...transformContext,
|
|
806
844
|
forceQuantizeDecision: chunkPlan.forceQuantizeDecision,
|
|
845
|
+
originalTensorShape: tensor.shape,
|
|
807
846
|
});
|
|
808
847
|
const normalized = normalizeWorkerTransformResult(transformed, chunkTensor);
|
|
809
848
|
processedBytes += chunkTensorData.byteLength;
|
|
@@ -839,10 +878,36 @@ function createNodeTensorTransformer(options) {
|
|
|
839
878
|
outputOffset += chunkResult.tensorData.byteLength;
|
|
840
879
|
}
|
|
841
880
|
|
|
881
|
+
const companionResults = chunkResults.filter((chunkResult) => (
|
|
882
|
+
chunkResult.companionData instanceof Uint8Array
|
|
883
|
+
));
|
|
884
|
+
let companionData = null;
|
|
885
|
+
let sourceTransform = null;
|
|
886
|
+
if (companionResults.length > 0) {
|
|
887
|
+
if (companionResults.length !== chunkResults.length) {
|
|
888
|
+
throw new Error(`node convert: inconsistent chunk companion data for ${tensor.name}.`);
|
|
889
|
+
}
|
|
890
|
+
sourceTransform = chunkResults[0].sourceTransform ?? null;
|
|
891
|
+
if (!sourceTransform) {
|
|
892
|
+
throw new Error(`node convert: chunk companion data is missing sourceTransform for ${tensor.name}.`);
|
|
893
|
+
}
|
|
894
|
+
const totalCompanionBytes = companionResults.reduce((sum, chunkResult) => (
|
|
895
|
+
sum + chunkResult.companionData.byteLength
|
|
896
|
+
), 0);
|
|
897
|
+
companionData = new Uint8Array(totalCompanionBytes);
|
|
898
|
+
let companionOffset = 0;
|
|
899
|
+
for (const chunkResult of companionResults) {
|
|
900
|
+
companionData.set(chunkResult.companionData, companionOffset);
|
|
901
|
+
companionOffset += chunkResult.companionData.byteLength;
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
|
|
842
905
|
return {
|
|
843
906
|
tensorData: combined,
|
|
844
907
|
outDtype,
|
|
845
908
|
outLayout,
|
|
909
|
+
...(companionData ? { companionData } : {}),
|
|
910
|
+
...(sourceTransform ? { sourceTransform } : {}),
|
|
846
911
|
};
|
|
847
912
|
};
|
|
848
913
|
}
|
|
@@ -933,6 +998,7 @@ function createNodeLargeTensorTransformer(options) {
|
|
|
933
998
|
const transformed = await pool.transformTensor(chunkTensor, chunkTensorData, {
|
|
934
999
|
...transformContext,
|
|
935
1000
|
forceQuantizeDecision: chunkPlan.forceQuantizeDecision,
|
|
1001
|
+
originalTensorShape: tensor.shape,
|
|
936
1002
|
});
|
|
937
1003
|
const normalized = normalizeWorkerTransformResult(transformed, chunkTensor);
|
|
938
1004
|
if (outDtype == null) {
|
|
@@ -1009,6 +1075,9 @@ export async function convertSafetensorsDirectory(options) {
|
|
|
1009
1075
|
import('../storage/shard-manager.js'),
|
|
1010
1076
|
]);
|
|
1011
1077
|
|
|
1078
|
+
const hashStringSha256 = async (value) => (
|
|
1079
|
+
computeHash(new TextEncoder().encode(String(value)), 'sha256')
|
|
1080
|
+
);
|
|
1012
1081
|
const converterConfig = createConverterConfig(converterConfigOverride ?? undefined);
|
|
1013
1082
|
const executionConfig = normalizeExecutionConfig(
|
|
1014
1083
|
options?.execution,
|
|
@@ -1324,6 +1393,10 @@ export async function convertSafetensorsDirectory(options) {
|
|
|
1324
1393
|
quantization: targetQuantization,
|
|
1325
1394
|
tokenizerJson,
|
|
1326
1395
|
tokenizerConfig,
|
|
1396
|
+
// GGUF inputs carry the tokenizer (eos/bos/pad IDs, chat template, etc.)
|
|
1397
|
+
// inside config.tokenizer; lift it to model.tokenizer so resolveEosTokenId
|
|
1398
|
+
// and other manifest helpers find it without a GGUF-specific branch.
|
|
1399
|
+
tokenizer: tokenizerConfig ?? config?.tokenizer ?? null,
|
|
1327
1400
|
generationConfig,
|
|
1328
1401
|
tokenizerModel: hasTokenizerModel ? 'tokenizer.model' : null,
|
|
1329
1402
|
embeddingPostprocessor,
|
|
@@ -1429,6 +1502,12 @@ export async function convertSafetensorsDirectory(options) {
|
|
|
1429
1502
|
architecture: manifestArchitecture,
|
|
1430
1503
|
inference,
|
|
1431
1504
|
converterConfig,
|
|
1505
|
+
source: pathToFileURL(inputDir).href,
|
|
1506
|
+
sourcePath: inputDir,
|
|
1507
|
+
sourceFormat: isInputGgufFile ? 'gguf' : 'safetensors',
|
|
1508
|
+
conversionConfigPath: toRepoRelativePath(options?.configPath),
|
|
1509
|
+
conversionConfig: converterConfigOverride ?? null,
|
|
1510
|
+
hashString: hashStringSha256,
|
|
1432
1511
|
tensorTransformer,
|
|
1433
1512
|
largeTensorTransformer,
|
|
1434
1513
|
onProgress(update) {
|
|
@@ -1463,6 +1542,16 @@ export async function convertSafetensorsDirectory(options) {
|
|
|
1463
1542
|
if (!deferredManifestState.manifest) {
|
|
1464
1543
|
throw new Error('node convert: convert core did not produce a manifest.');
|
|
1465
1544
|
}
|
|
1545
|
+
const builtIntegrity = await buildManifestIntegrityFromModelDir(result.manifest, {
|
|
1546
|
+
modelDir: outputDir,
|
|
1547
|
+
tensorMap: result.manifest.tensors ?? undefined,
|
|
1548
|
+
readRange: fileRangeReader.readRange,
|
|
1549
|
+
});
|
|
1550
|
+
result.manifest = {
|
|
1551
|
+
...result.manifest,
|
|
1552
|
+
integrityExtensions: builtIntegrity.integrityExtensions,
|
|
1553
|
+
};
|
|
1554
|
+
deferredManifestState.manifest = result.manifest;
|
|
1466
1555
|
await io.writeManifest(result.manifest);
|
|
1467
1556
|
|
|
1468
1557
|
const report = buildConvertReport(result, {
|
|
@@ -15,6 +15,7 @@ import { parseSafetensorsHeader } from '../formats/safetensors/types.js';
|
|
|
15
15
|
import { parseTFLiteFromSource } from '../formats/tflite/types.js';
|
|
16
16
|
import { log } from '../debug/index.js';
|
|
17
17
|
import { formatBytes } from '../storage/quota.js';
|
|
18
|
+
import { toArrayBuffer } from '../utils/array-buffer.js';
|
|
18
19
|
import {
|
|
19
20
|
createSourceStorageContext,
|
|
20
21
|
} from './source-runtime-bundle.js';
|
|
@@ -30,16 +31,6 @@ import {
|
|
|
30
31
|
} from './litert-package-runtime.js';
|
|
31
32
|
const MAX_NODE_READ_BYTES = 64 * 1024 * 1024;
|
|
32
33
|
|
|
33
|
-
function toArrayBuffer(value, label) {
|
|
34
|
-
if (value instanceof ArrayBuffer) {
|
|
35
|
-
return value;
|
|
36
|
-
}
|
|
37
|
-
if (value instanceof Uint8Array) {
|
|
38
|
-
return value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength);
|
|
39
|
-
}
|
|
40
|
-
throw new Error(`${label} must return ArrayBuffer or Uint8Array.`);
|
|
41
|
-
}
|
|
42
|
-
|
|
43
34
|
function normalizePath(value) {
|
|
44
35
|
return String(value || '').trim();
|
|
45
36
|
}
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
import { downloadModel, estimateTimeRemaining, formatSpeed } from '../storage/downloader.js';
|
|
9
9
|
import { isOPFSAvailable, formatBytes } from '../storage/quota.js';
|
|
10
10
|
import { parseManifest, getManifestUrl } from '../formats/rdrr/index.js';
|
|
11
|
+
import { cloneJsonValue } from '../utils/clone-json.js';
|
|
11
12
|
import { log } from '../debug/index.js';
|
|
12
13
|
import {
|
|
13
14
|
resolveSourceArtifact,
|
|
@@ -47,10 +48,6 @@ function hasSameShardSet(aManifest, bManifest) {
|
|
|
47
48
|
return true;
|
|
48
49
|
}
|
|
49
50
|
|
|
50
|
-
function cloneJsonValue(value) {
|
|
51
|
-
return value == null ? value : JSON.parse(JSON.stringify(value));
|
|
52
|
-
}
|
|
53
|
-
|
|
54
51
|
function preserveCachedSourceRuntimeMetadata(remoteManifest, cachedManifest) {
|
|
55
52
|
const cachedSourceRuntime = cachedManifest?.metadata?.sourceRuntime;
|
|
56
53
|
if (!cachedSourceRuntime || typeof cachedSourceRuntime !== 'object') {
|