doppler-gpu 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +19 -15
- package/package.json +77 -29
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +64 -6
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +524 -350
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +12 -2
- package/src/config/schema/doppler.schema.js +108 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +68 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +76 -0
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +107 -0
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +3 -21
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +6 -5
- package/src/gpu/kernels/index.js +3 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +12 -0
- package/src/inference/browser-harness-text-helpers.js +223 -14
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +264 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +22 -27
- package/src/inference/pipelines/text/embed.js +0 -39
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +4 -5
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +8 -3
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +223 -547
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +5 -2
- package/src/inference/pipelines/text/init.js +183 -724
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.js +2 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +120 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +223 -27
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +13 -3
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +11 -3
- package/src/storage/artifact-storage-context.js +231 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +21 -24
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +2 -0
- package/src/tooling/source-runtime-bundle.js +3 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -94,7 +94,31 @@ export function resolveWeightLayout(location) {
|
|
|
94
94
|
});
|
|
95
95
|
}
|
|
96
96
|
|
|
97
|
-
export function
|
|
97
|
+
export function resolveLargeWeightOverrides(manifestOverrides, runtimeOverrides) {
|
|
98
|
+
if (Array.isArray(runtimeOverrides)) {
|
|
99
|
+
return runtimeOverrides;
|
|
100
|
+
}
|
|
101
|
+
if (Array.isArray(manifestOverrides)) {
|
|
102
|
+
return manifestOverrides;
|
|
103
|
+
}
|
|
104
|
+
return runtimeOverrides ?? null;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export function shouldStreamLargeWeight(name, location, label, gpuCapabilities, keepF32Weights, manifestOverrides = null) {
|
|
108
|
+
// Runtime profiles may explicitly replace manifest residency overrides with
|
|
109
|
+
// an array, including [] to request ordinary large-weight streaming policy.
|
|
110
|
+
const overrides = resolveLargeWeightOverrides(
|
|
111
|
+
manifestOverrides,
|
|
112
|
+
getLargeWeightConfig().gpuResidentOverrides
|
|
113
|
+
);
|
|
114
|
+
if (Array.isArray(overrides) && overrides.includes(name)) {
|
|
115
|
+
log.info(
|
|
116
|
+
'Loader',
|
|
117
|
+
`${label} weight "${name}" forced GPU-resident via inference.largeWeights.gpuResidentOverrides.`
|
|
118
|
+
);
|
|
119
|
+
return false;
|
|
120
|
+
}
|
|
121
|
+
|
|
98
122
|
const maxBytes = getLargeWeightMaxBytes();
|
|
99
123
|
if (!maxBytes) return false;
|
|
100
124
|
|
|
@@ -45,11 +45,6 @@ export interface MemorySnapshot {
|
|
|
45
45
|
};
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
-
export interface MemoryMonitorState {
|
|
49
|
-
startTime: number;
|
|
50
|
-
interval: ReturnType<typeof setInterval> | null;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
48
|
// ============================================================================
|
|
54
49
|
// Memory Snapshot
|
|
55
50
|
// ============================================================================
|
|
@@ -118,3 +113,20 @@ export declare class MemoryMonitor {
|
|
|
118
113
|
*/
|
|
119
114
|
getElapsed(): number;
|
|
120
115
|
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Rolling time-series buffer of memory snapshots. Captures samples on
|
|
119
|
+
* a fixed interval and exposes them for post-run analysis.
|
|
120
|
+
*/
|
|
121
|
+
export declare class MemoryTimeSeries {
|
|
122
|
+
constructor(sampleIntervalMs?: number);
|
|
123
|
+
|
|
124
|
+
/** Begin sampling; replaces any existing interval. */
|
|
125
|
+
start(): void;
|
|
126
|
+
|
|
127
|
+
/** Stop sampling and finalize the sample buffer. */
|
|
128
|
+
stop(): void;
|
|
129
|
+
|
|
130
|
+
/** Samples collected since the last start(). */
|
|
131
|
+
getSamples(): Array<{ timestamp: number; snapshot: MemorySnapshot }>;
|
|
132
|
+
}
|
|
@@ -21,3 +21,9 @@ export declare const Q8_0_BLOCK_BYTES: number;
|
|
|
21
21
|
|
|
22
22
|
/** Q8_0 block size in elements (32 weights per block) */
|
|
23
23
|
export declare const Q8_0_BLOCK_SIZE: number;
|
|
24
|
+
|
|
25
|
+
/** Pad an element count up to the nearest Q4K block boundary. */
|
|
26
|
+
export declare function padToQ4KBlock(size: number): number;
|
|
27
|
+
|
|
28
|
+
/** Number of Q4K blocks required to cover `numElements`. */
|
|
29
|
+
export declare function q4kBlockCount(numElements: number): number;
|
|
@@ -4,6 +4,7 @@ import type { TensorLocation } from './loader-types.js';
|
|
|
4
4
|
export interface BuildTensorLocationsOptions {
|
|
5
5
|
hasCustomLoader?: boolean;
|
|
6
6
|
tensorsJsonUrl?: string | null;
|
|
7
|
+
loadTensorsJson?: (() => Promise<string | Record<string, unknown> | null | undefined>) | null;
|
|
7
8
|
}
|
|
8
9
|
|
|
9
10
|
export function buildTensorLocations(
|
|
@@ -61,6 +61,21 @@ export async function buildTensorLocations(manifest, options = {}) {
|
|
|
61
61
|
}
|
|
62
62
|
|
|
63
63
|
// 2. Try HTTP if we have a tensors URL set (for HTTP-based testing)
|
|
64
|
+
if (!tensorsJsonRaw && typeof options.loadTensorsJson === 'function') {
|
|
65
|
+
try {
|
|
66
|
+
const payload = await options.loadTensorsJson();
|
|
67
|
+
if (payload != null) {
|
|
68
|
+
tensorsJsonRaw = typeof payload === 'string' ? payload : JSON.stringify(payload);
|
|
69
|
+
trace.push({ source: 'storage-context', outcome: 'resolved' });
|
|
70
|
+
} else {
|
|
71
|
+
trace.push({ source: 'storage-context', outcome: 'not-found' });
|
|
72
|
+
}
|
|
73
|
+
} catch (e) {
|
|
74
|
+
trace.push({ source: 'storage-context', outcome: `error: ${e.message}` });
|
|
75
|
+
log.warn('Loader', `Failed to load tensors.json from storage context: ${e.message}`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
64
79
|
if (!tensorsJsonRaw && options.tensorsJsonUrl) {
|
|
65
80
|
try {
|
|
66
81
|
const resp = await fetch(options.tensorsJsonUrl);
|
|
@@ -760,8 +760,13 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
760
760
|
);
|
|
761
761
|
}
|
|
762
762
|
|
|
763
|
+
// scaleBytes was loaded for the `[rowStart, rowStart + rowCount)` window,
|
|
764
|
+
// so its byte offsets are relative to `rowStart`. Subtract `rowStart`
|
|
765
|
+
// before indexing into scaleView; keep the caller contract of passing
|
|
766
|
+
// absolute row indices so error messages still name the real row.
|
|
763
767
|
const resolveScale = (logicalRow, col = 0) => {
|
|
764
|
-
const
|
|
768
|
+
const localRow = logicalRow - rowStart;
|
|
769
|
+
const rowScaleOffset = localRow * spec.scaleRowBytes;
|
|
765
770
|
const storedScale = scaleCompanion
|
|
766
771
|
? (scaleView.getUint8(rowScaleOffset + Math.min(Math.floor(col / scaleSubBlockWidth), scaleSubCount - 1))
|
|
767
772
|
- scaleCompanion.zeroPoint) * scaleCompanion.scale
|
|
@@ -19,6 +19,8 @@ export interface TensorLoadConfig {
|
|
|
19
19
|
useFusedQ4K: boolean;
|
|
20
20
|
/** Q4K weight materialization mode derived from the resolved execution graph */
|
|
21
21
|
q4kMaterializationMode?: 'dense' | 'fused' | 'mixed';
|
|
22
|
+
/** Tensor roles that require raw Q4K buffers because their graph step pins a fused Q4K kernel */
|
|
23
|
+
q4kFusedRoles?: string[];
|
|
22
24
|
/** Debug controls for Q4K loading/dequantization */
|
|
23
25
|
loaderDebug?: LoaderDebugConfigSchema | null;
|
|
24
26
|
/** Keep weights as F32 (disable F16 downcasting) */
|
|
@@ -99,15 +101,6 @@ export declare function loadQ4KDequant(
|
|
|
99
101
|
config: TensorLoadConfig
|
|
100
102
|
): Promise<TensorLoadResult>;
|
|
101
103
|
|
|
102
|
-
/**
|
|
103
|
-
* Load Q4K tensor to GPU with both dense and fused-compatible materializations.
|
|
104
|
-
*/
|
|
105
|
-
export declare function loadQ4KMixed(
|
|
106
|
-
shardData: Uint8Array,
|
|
107
|
-
location: TensorLocation,
|
|
108
|
-
name: string,
|
|
109
|
-
config: TensorLoadConfig
|
|
110
|
-
): Promise<TensorLoadResult>;
|
|
111
104
|
|
|
112
105
|
/**
|
|
113
106
|
* Load Q6K tensor to GPU.
|
|
@@ -17,6 +17,7 @@ import { hasSourceTransform } from './source-transform.js';
|
|
|
17
17
|
// ============================================================================
|
|
18
18
|
|
|
19
19
|
let loggedF32UpcastNonMatmul = false;
|
|
20
|
+
let loggedQ4KLimitFallback = false;
|
|
20
21
|
|
|
21
22
|
function isGpuBufferInstance(value) {
|
|
22
23
|
return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
|
|
@@ -128,7 +129,10 @@ function isEmbeddingRole(location) {
|
|
|
128
129
|
|
|
129
130
|
export function shouldUseFusedQ4K(location, config) {
|
|
130
131
|
if (!config.useFusedQ4K) return false;
|
|
132
|
+
return canUseFusedQ4KStorage(location, config);
|
|
133
|
+
}
|
|
131
134
|
|
|
135
|
+
function canUseFusedQ4KStorage(location, config) {
|
|
132
136
|
const caps = config.gpuCapabilities || getKernelCapabilities();
|
|
133
137
|
if (!caps?.hasSubgroups) return false;
|
|
134
138
|
|
|
@@ -141,6 +145,75 @@ export function shouldUseFusedQ4K(location, config) {
|
|
|
141
145
|
return true;
|
|
142
146
|
}
|
|
143
147
|
|
|
148
|
+
function getQ4KDenseMaterializedSizeBytes(location, config) {
|
|
149
|
+
if (!Array.isArray(location.shape) || location.shape.length === 0) {
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
const elementCount = getShapeElementCount(location.shape);
|
|
153
|
+
if (!Number.isFinite(elementCount) || elementCount <= 0) {
|
|
154
|
+
return null;
|
|
155
|
+
}
|
|
156
|
+
const outputDtype = getQ4KOutputDtype(location, config);
|
|
157
|
+
const bytesPerElement = outputDtype === 'f16' ? 2 : 4;
|
|
158
|
+
return elementCount * bytesPerElement;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function getMaxStorageBufferBindingSize() {
|
|
162
|
+
const device = getDevice();
|
|
163
|
+
const maxStorage = device?.limits?.maxStorageBufferBindingSize;
|
|
164
|
+
return Number.isFinite(maxStorage) && maxStorage > 0 ? maxStorage : null;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function resolveQ4KLimitFallback(location, config) {
|
|
168
|
+
if (location?.dtype !== 'Q4_K_M' && location?.dtype !== 'Q4_K') {
|
|
169
|
+
return {
|
|
170
|
+
denseExceedsBindingLimit: false,
|
|
171
|
+
limitFallbackEligible: false,
|
|
172
|
+
denseSizeBytes: null,
|
|
173
|
+
maxBindingSizeBytes: null,
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const denseSizeBytes = getQ4KDenseMaterializedSizeBytes(location, config);
|
|
178
|
+
const maxBindingSizeBytes = getMaxStorageBufferBindingSize();
|
|
179
|
+
const denseExceedsBindingLimit = (
|
|
180
|
+
denseSizeBytes != null
|
|
181
|
+
&& maxBindingSizeBytes != null
|
|
182
|
+
&& denseSizeBytes > maxBindingSizeBytes
|
|
183
|
+
);
|
|
184
|
+
const packedSizeBytes = Number.isFinite(location.size) ? location.size : null;
|
|
185
|
+
const packedFitsBindingLimit = (
|
|
186
|
+
packedSizeBytes != null
|
|
187
|
+
&& maxBindingSizeBytes != null
|
|
188
|
+
&& packedSizeBytes <= maxBindingSizeBytes
|
|
189
|
+
);
|
|
190
|
+
const limitFallbackEligible = (
|
|
191
|
+
denseExceedsBindingLimit
|
|
192
|
+
&& config.keepF32Weights !== true
|
|
193
|
+
&& packedFitsBindingLimit
|
|
194
|
+
&& canUseFusedQ4KStorage(location, config)
|
|
195
|
+
);
|
|
196
|
+
|
|
197
|
+
return {
|
|
198
|
+
denseExceedsBindingLimit,
|
|
199
|
+
limitFallbackEligible,
|
|
200
|
+
denseSizeBytes,
|
|
201
|
+
maxBindingSizeBytes,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function logQ4KLimitFallbackOnce(name, fallback) {
|
|
206
|
+
if (loggedQ4KLimitFallback) {
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
loggedQ4KLimitFallback = true;
|
|
210
|
+
log.warn(
|
|
211
|
+
'Loader',
|
|
212
|
+
`Q4K dense materialization for "${name}" would require ${fallback.denseSizeBytes} bytes, ` +
|
|
213
|
+
`exceeding maxStorageBufferBindingSize=${fallback.maxBindingSizeBytes}; retaining packed Q4K for fused matmul.`
|
|
214
|
+
);
|
|
215
|
+
}
|
|
216
|
+
|
|
144
217
|
// ============================================================================
|
|
145
218
|
// Dtype Output Selection
|
|
146
219
|
// ============================================================================
|
|
@@ -389,7 +462,7 @@ export async function loadQ4KDequant(shardData, location, name, config) {
|
|
|
389
462
|
}
|
|
390
463
|
}
|
|
391
464
|
|
|
392
|
-
|
|
465
|
+
async function loadQ4KMixed(shardData, location, name, config) {
|
|
393
466
|
const canMaterializeMixed = shouldUseFusedQ4K(location, config)
|
|
394
467
|
&& config.q4kLayout === 'row';
|
|
395
468
|
if (!canMaterializeMixed) {
|
|
@@ -693,22 +766,32 @@ const GPU_LOADER_DISPATCH = {
|
|
|
693
766
|
export async function loadTensorToGPU(shardData, location, name, config) {
|
|
694
767
|
const dtype = location.dtype;
|
|
695
768
|
const useFusedQ4K = shouldUseFusedQ4K(location, config);
|
|
769
|
+
const requiresFusedQ4KRole = Array.isArray(config?.q4kFusedRoles)
|
|
770
|
+
&& config.q4kFusedRoles.includes(location.role);
|
|
696
771
|
const caps = config?.gpuCapabilities || getKernelCapabilities();
|
|
697
772
|
const platformId = getPlatformConfig()?.platform?.id ?? null;
|
|
698
773
|
const q4kReferenceContext = getQ4KCpuReferenceContext(shardData, location, config);
|
|
699
774
|
const q4kBasicBackendClass = platformId === 'basic'
|
|
700
775
|
|| (caps?.hasSubgroups !== true && caps?.hasF16 !== true);
|
|
776
|
+
const q4kLimitFallback = resolveQ4KLimitFallback(location, config);
|
|
701
777
|
const loaderPath = selectRuleValue('loader', 'tensorLoader', 'gpuLoaderPath', {
|
|
702
778
|
dtype,
|
|
779
|
+
role: location.role ?? null,
|
|
703
780
|
useFusedQ4K,
|
|
781
|
+
requiresFusedQ4KRole,
|
|
704
782
|
q4kMaterializationMode: config.q4kMaterializationMode ?? 'dense',
|
|
705
783
|
q4kCpuReferenceEligible: q4kReferenceContext.eligible,
|
|
706
784
|
q4kBasicBackendClass,
|
|
785
|
+
q4kDenseExceedsBindingLimit: q4kLimitFallback.denseExceedsBindingLimit,
|
|
786
|
+
q4kLimitFallbackEligible: q4kLimitFallback.limitFallbackEligible,
|
|
707
787
|
});
|
|
708
788
|
const loader = GPU_LOADER_DISPATCH[loaderPath];
|
|
709
789
|
if (!loader) {
|
|
710
790
|
throw new Error(`Unknown GPU loader path: "${loaderPath}" for dtype "${dtype}"`);
|
|
711
791
|
}
|
|
792
|
+
if (loaderPath === 'q4k_fused' && q4kLimitFallback.limitFallbackEligible) {
|
|
793
|
+
logQ4KLimitFallbackOnce(name, q4kLimitFallback);
|
|
794
|
+
}
|
|
712
795
|
return loader(shardData, location, name, config);
|
|
713
796
|
}
|
|
714
797
|
|
|
@@ -12,14 +12,6 @@
|
|
|
12
12
|
* @module memory/address-table
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
/**
|
|
16
|
-
* Decoded address parts
|
|
17
|
-
*/
|
|
18
|
-
export interface DecodedAddress {
|
|
19
|
-
segmentIndex: number;
|
|
20
|
-
offset: number;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
15
|
/**
|
|
24
16
|
* Address range chunk for split operations
|
|
25
17
|
*/
|
|
@@ -63,7 +55,7 @@ export declare class AddressTable {
|
|
|
63
55
|
* Decode virtual address into segment index and offset
|
|
64
56
|
* @param virtualAddress - Virtual address to decode
|
|
65
57
|
*/
|
|
66
|
-
decode(virtualAddress: number):
|
|
58
|
+
decode(virtualAddress: number): { segmentIndex: number; offset: number };
|
|
67
59
|
|
|
68
60
|
/**
|
|
69
61
|
* Get the segment index from a virtual address
|
|
@@ -170,11 +170,6 @@ export declare class BufferPool {
|
|
|
170
170
|
*/
|
|
171
171
|
export function getBufferPool(): BufferPool;
|
|
172
172
|
|
|
173
|
-
/**
|
|
174
|
-
* Create a standalone buffer pool
|
|
175
|
-
*/
|
|
176
|
-
export function createBufferPool(debugMode?: boolean, schemaConfig?: BufferPoolConfigSchema): BufferPool;
|
|
177
|
-
|
|
178
173
|
/**
|
|
179
174
|
* Destroy the global buffer pool
|
|
180
175
|
*/
|
|
@@ -182,14 +177,12 @@ export function destroyBufferPool(): void;
|
|
|
182
177
|
|
|
183
178
|
// Convenience exports for common operations
|
|
184
179
|
export declare const createStagingBuffer: (size: number) => GPUBuffer;
|
|
185
|
-
export declare const createUploadBuffer: (size: number) => GPUBuffer;
|
|
186
180
|
export declare const createUniformBuffer: (size: number) => GPUBuffer;
|
|
187
181
|
export declare const acquireBuffer: (size: number, usage?: GPUBufferUsageFlags, label?: string) => GPUBuffer;
|
|
188
182
|
export declare const releaseBuffer: (buffer: GPUBuffer) => void;
|
|
189
183
|
export declare const discardBuffer: (buffer: GPUBuffer) => void;
|
|
190
184
|
export declare const isBufferActive: (buffer: GPUBuffer) => boolean;
|
|
191
185
|
export declare function markPersistentBuffer(buffer: GPUBuffer): void;
|
|
192
|
-
export declare function unmarkPersistentBuffer(buffer: GPUBuffer): void;
|
|
193
186
|
export declare function isPersistentBuffer(buffer: GPUBuffer): boolean;
|
|
194
187
|
export declare class PersistentBufferSet extends Set<GPUBuffer> {}
|
|
195
188
|
export declare const getBufferRequestedSize: (buffer: GPUBuffer) => number;
|
|
@@ -197,12 +190,3 @@ export declare const uploadData: (buffer: GPUBuffer, data: ArrayBuffer | ArrayBu
|
|
|
197
190
|
export declare const readBuffer: (buffer: GPUBuffer, size?: number) => Promise<ArrayBuffer>;
|
|
198
191
|
export declare const readBufferSlice: (buffer: GPUBuffer, offset: number, size: number) => Promise<ArrayBuffer>;
|
|
199
192
|
export declare const forceBufferPoolReclaim: (targetRatio?: number | null) => void;
|
|
200
|
-
|
|
201
|
-
/**
|
|
202
|
-
* Scoped buffer helper - automatically releases buffer when done
|
|
203
|
-
*/
|
|
204
|
-
export function withBuffer<T>(
|
|
205
|
-
size: number,
|
|
206
|
-
usage: GPUBufferUsageFlags,
|
|
207
|
-
fn: (buffer: GPUBuffer) => Promise<T>
|
|
208
|
-
): Promise<T>;
|
|
@@ -794,14 +794,6 @@ export function getBufferPool() {
|
|
|
794
794
|
}
|
|
795
795
|
|
|
796
796
|
|
|
797
|
-
export function createBufferPool(debugMode, schemaConfig) {
|
|
798
|
-
if (!schemaConfig) {
|
|
799
|
-
throw new Error('createBufferPool requires schemaConfig from runtime.shared.bufferPool.');
|
|
800
|
-
}
|
|
801
|
-
return new BufferPool(debugMode, schemaConfig);
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
|
|
805
797
|
export function destroyBufferPool() {
|
|
806
798
|
if (globalPool) {
|
|
807
799
|
globalPool.destroy();
|
|
@@ -814,8 +806,6 @@ export function destroyBufferPool() {
|
|
|
814
806
|
|
|
815
807
|
export const createStagingBuffer = (size) => getBufferPool().createStagingBuffer(size);
|
|
816
808
|
|
|
817
|
-
export const createUploadBuffer = (size) => getBufferPool().createUploadBuffer(size);
|
|
818
|
-
|
|
819
809
|
export const createUniformBuffer = (size) => getBufferPool().createUniformBuffer(size);
|
|
820
810
|
|
|
821
811
|
export const acquireBuffer = (size, usage, label) =>
|
|
@@ -823,15 +813,6 @@ export const acquireBuffer = (size, usage, label) =>
|
|
|
823
813
|
|
|
824
814
|
export const releaseBuffer = (buffer) => getBufferPool().release(buffer);
|
|
825
815
|
|
|
826
|
-
/**
|
|
827
|
-
* Safe release helper: no-op if buffer is null or undefined.
|
|
828
|
-
* Use in error/cleanup paths where the buffer may not have been acquired.
|
|
829
|
-
*/
|
|
830
|
-
export function safeRelease(buffer) {
|
|
831
|
-
if (buffer == null) return;
|
|
832
|
-
getBufferPool().release(buffer);
|
|
833
|
-
}
|
|
834
|
-
|
|
835
816
|
export const discardBuffer = (buffer) => getBufferPool().discard(buffer);
|
|
836
817
|
|
|
837
818
|
export const isBufferActive = (buffer) =>
|
|
@@ -843,7 +824,7 @@ export function markPersistentBuffer(buffer) {
|
|
|
843
824
|
}
|
|
844
825
|
}
|
|
845
826
|
|
|
846
|
-
|
|
827
|
+
function unmarkPersistentBuffer(buffer) {
|
|
847
828
|
if (buffer && typeof buffer === 'object') {
|
|
848
829
|
persistentBuffers.delete(buffer);
|
|
849
830
|
}
|
|
@@ -888,16 +869,3 @@ export const forceBufferPoolReclaim = (targetRatio) =>
|
|
|
888
869
|
getBufferPool().forceReclaim(targetRatio);
|
|
889
870
|
|
|
890
871
|
|
|
891
|
-
export async function withBuffer(
|
|
892
|
-
size,
|
|
893
|
-
usage,
|
|
894
|
-
fn
|
|
895
|
-
) {
|
|
896
|
-
const pool = getBufferPool();
|
|
897
|
-
const buffer = pool.acquire(size, usage);
|
|
898
|
-
try {
|
|
899
|
-
return await fn(buffer);
|
|
900
|
-
} finally {
|
|
901
|
-
pool.release(buffer);
|
|
902
|
-
}
|
|
903
|
-
}
|
|
@@ -9,47 +9,7 @@
|
|
|
9
9
|
* @module memory/heap-manager
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
-
import { MemoryStrategy
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* Memory segment for segmented heap mode
|
|
16
|
-
*/
|
|
17
|
-
export interface MemorySegment {
|
|
18
|
-
index: number;
|
|
19
|
-
buffer: ArrayBuffer;
|
|
20
|
-
used: number;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
/**
|
|
24
|
-
* Allocation result
|
|
25
|
-
*/
|
|
26
|
-
export interface AllocationResult {
|
|
27
|
-
/** Virtual address for this allocation */
|
|
28
|
-
virtualAddress: number;
|
|
29
|
-
/** Allocated size in bytes */
|
|
30
|
-
size: number;
|
|
31
|
-
/** Direct view into the buffer */
|
|
32
|
-
view: Uint8Array;
|
|
33
|
-
/** Which strategy was used */
|
|
34
|
-
strategy: MemoryStrategy;
|
|
35
|
-
/** Segment index (segmented only) */
|
|
36
|
-
segmentIndex?: number;
|
|
37
|
-
/** Offset within segment (segmented only) */
|
|
38
|
-
segmentOffset?: number;
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
/**
|
|
42
|
-
* Heap manager statistics
|
|
43
|
-
*/
|
|
44
|
-
export interface HeapStats {
|
|
45
|
-
strategy: MemoryStrategy | null;
|
|
46
|
-
totalAllocated: number;
|
|
47
|
-
segmentCount: number;
|
|
48
|
-
memory64HeapSize: number;
|
|
49
|
-
allocated?: number;
|
|
50
|
-
limit?: number;
|
|
51
|
-
maxSize?: number;
|
|
52
|
-
}
|
|
12
|
+
import { MemoryStrategy } from './capability.js';
|
|
53
13
|
|
|
54
14
|
/**
|
|
55
15
|
* HeapManager - Unified interface for both memory strategies
|
|
@@ -64,7 +24,14 @@ export declare class HeapManager {
|
|
|
64
24
|
* Allocate buffer for model data
|
|
65
25
|
* @param size - Size in bytes
|
|
66
26
|
*/
|
|
67
|
-
allocate(size: number):
|
|
27
|
+
allocate(size: number): {
|
|
28
|
+
virtualAddress: number;
|
|
29
|
+
size: number;
|
|
30
|
+
view: Uint8Array;
|
|
31
|
+
strategy: MemoryStrategy;
|
|
32
|
+
segmentIndex?: number;
|
|
33
|
+
segmentOffset?: number;
|
|
34
|
+
};
|
|
68
35
|
|
|
69
36
|
/**
|
|
70
37
|
* Read data from virtual address
|
|
@@ -90,7 +57,15 @@ export declare class HeapManager {
|
|
|
90
57
|
/**
|
|
91
58
|
* Get memory stats
|
|
92
59
|
*/
|
|
93
|
-
getStats():
|
|
60
|
+
getStats(): {
|
|
61
|
+
strategy: MemoryStrategy | null;
|
|
62
|
+
totalAllocated: number;
|
|
63
|
+
segmentCount: number;
|
|
64
|
+
memory64HeapSize: number;
|
|
65
|
+
allocated?: number;
|
|
66
|
+
limit?: number;
|
|
67
|
+
maxSize?: number;
|
|
68
|
+
};
|
|
94
69
|
|
|
95
70
|
/**
|
|
96
71
|
* Free all memory (for model unload)
|
|
@@ -10,45 +10,30 @@
|
|
|
10
10
|
* @module memory/unified-detect
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
/**
|
|
14
|
-
* Apple Silicon detection result
|
|
15
|
-
*/
|
|
16
|
-
export interface AppleSiliconInfo {
|
|
17
|
-
isApple: boolean;
|
|
18
|
-
mSeriesGen?: number | null;
|
|
19
|
-
vendor?: string;
|
|
20
|
-
device?: string;
|
|
21
|
-
description?: string;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* AMD unified memory detection result
|
|
26
|
-
*/
|
|
27
|
-
export interface AMDUnifiedInfo {
|
|
28
|
-
isAMDUnified: boolean;
|
|
29
|
-
isStrix?: boolean;
|
|
30
|
-
vendor?: string;
|
|
31
|
-
device?: string;
|
|
32
|
-
description?: string;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
/**
|
|
36
|
-
* WebGPU buffer limit indicators
|
|
37
|
-
*/
|
|
38
|
-
export interface LimitIndicators {
|
|
39
|
-
largeBuffers: boolean;
|
|
40
|
-
maxBufferSize?: number;
|
|
41
|
-
maxStorageBufferBindingSize?: number;
|
|
42
|
-
}
|
|
43
|
-
|
|
44
13
|
/**
|
|
45
14
|
* Unified memory detection result
|
|
46
15
|
*/
|
|
47
16
|
export interface UnifiedMemoryInfo {
|
|
48
17
|
isUnified: boolean;
|
|
49
|
-
apple?:
|
|
50
|
-
|
|
51
|
-
|
|
18
|
+
apple?: {
|
|
19
|
+
isApple: boolean;
|
|
20
|
+
mSeriesGen?: number | null;
|
|
21
|
+
vendor?: string;
|
|
22
|
+
device?: string;
|
|
23
|
+
description?: string;
|
|
24
|
+
};
|
|
25
|
+
amd?: {
|
|
26
|
+
isAMDUnified: boolean;
|
|
27
|
+
isStrix?: boolean;
|
|
28
|
+
vendor?: string;
|
|
29
|
+
device?: string;
|
|
30
|
+
description?: string;
|
|
31
|
+
};
|
|
32
|
+
limits?: {
|
|
33
|
+
largeBuffers: boolean;
|
|
34
|
+
maxBufferSize?: number;
|
|
35
|
+
maxStorageBufferBindingSize?: number;
|
|
36
|
+
};
|
|
52
37
|
estimatedMemoryGB?: number | null;
|
|
53
38
|
reason: string;
|
|
54
39
|
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { FamilyModelEntry } from './family.d.ts';
|
|
2
|
+
|
|
3
|
+
export declare const FAMILY_ID: string;
|
|
4
|
+
export declare const HF_REPO_ID: string;
|
|
5
|
+
export declare const KNOWN_MODELS: ReadonlyArray<FamilyModelEntry>;
|
|
6
|
+
export declare function resolveModel(modelId: string): FamilyModelEntry | null;
|
|
7
|
+
export declare function resolveHfBaseUrl(modelId: string, revision?: string): string | null;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Static metadata for the EmbeddingGemma family of RDRR artifacts.
|
|
2
|
+
|
|
3
|
+
import { createFamily } from './family.js';
|
|
4
|
+
|
|
5
|
+
const family = createFamily({
|
|
6
|
+
familyId: 'embeddinggemma',
|
|
7
|
+
hfRepoId: 'Clocksmith/rdrr',
|
|
8
|
+
knownModels: [
|
|
9
|
+
{
|
|
10
|
+
modelId: 'google-embeddinggemma-300m-q4k-ehf16-af32',
|
|
11
|
+
label: 'EmbeddingGemma 300M (Q4K)',
|
|
12
|
+
sourceModel: 'google/embeddinggemma-300m',
|
|
13
|
+
hfPath: 'models/google-embeddinggemma-300m-q4k-ehf16-af32',
|
|
14
|
+
defaultRuntimeProfile: 'profiles/throughput',
|
|
15
|
+
modes: ['embedding'],
|
|
16
|
+
},
|
|
17
|
+
],
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
export const FAMILY_ID = family.FAMILY_ID;
|
|
21
|
+
export const HF_REPO_ID = family.HF_REPO_ID;
|
|
22
|
+
export const KNOWN_MODELS = family.KNOWN_MODELS;
|
|
23
|
+
export const resolveModel = family.resolveModel;
|
|
24
|
+
export const resolveHfBaseUrl = family.resolveHfBaseUrl;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export interface FamilyModelEntry {
|
|
2
|
+
readonly modelId: string;
|
|
3
|
+
readonly label: string;
|
|
4
|
+
readonly sourceModel: string;
|
|
5
|
+
readonly hfPath: string;
|
|
6
|
+
readonly defaultRuntimeProfile: string;
|
|
7
|
+
readonly modes: ReadonlyArray<'text' | 'vision' | 'embedding' | 'translate' | 'diffusion'>;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface FamilyModule {
|
|
11
|
+
readonly FAMILY_ID: string;
|
|
12
|
+
readonly HF_REPO_ID: string;
|
|
13
|
+
readonly KNOWN_MODELS: ReadonlyArray<FamilyModelEntry>;
|
|
14
|
+
resolveModel(modelId: string): FamilyModelEntry | null;
|
|
15
|
+
resolveHfBaseUrl(modelId: string, revision?: string): string | null;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface CreateFamilyInput {
|
|
19
|
+
readonly familyId: string;
|
|
20
|
+
readonly hfRepoId: string;
|
|
21
|
+
readonly knownModels: ReadonlyArray<FamilyModelEntry>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export declare function createFamily(input: CreateFamilyInput): FamilyModule;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// Shared factory for per-family pointer modules (qwen3, gemma3, gemma4,
|
|
2
|
+
// embeddinggemma). Centralizes the resolve helpers so each family file is
|
|
3
|
+
// pure data.
|
|
4
|
+
|
|
5
|
+
export function createFamily({ familyId, hfRepoId, knownModels }) {
|
|
6
|
+
const KNOWN_MODELS = Object.freeze(knownModels.map((m) => Object.freeze({ ...m, modes: Object.freeze([...m.modes]) })));
|
|
7
|
+
|
|
8
|
+
function resolveModel(modelId) {
|
|
9
|
+
return KNOWN_MODELS.find((m) => m.modelId === modelId) || null;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function resolveHfBaseUrl(modelId, revision = 'main') {
|
|
13
|
+
const entry = resolveModel(modelId);
|
|
14
|
+
if (!entry) return null;
|
|
15
|
+
return `https://huggingface.co/${hfRepoId}/resolve/${revision}/${entry.hfPath}`;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
return {
|
|
19
|
+
FAMILY_ID: familyId,
|
|
20
|
+
HF_REPO_ID: hfRepoId,
|
|
21
|
+
KNOWN_MODELS,
|
|
22
|
+
resolveModel,
|
|
23
|
+
resolveHfBaseUrl,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { FamilyModelEntry } from './family.d.ts';
|
|
2
|
+
|
|
3
|
+
export declare const FAMILY_ID: string;
|
|
4
|
+
export declare const HF_REPO_ID: string;
|
|
5
|
+
export declare const KNOWN_MODELS: ReadonlyArray<FamilyModelEntry>;
|
|
6
|
+
export declare function resolveModel(modelId: string): FamilyModelEntry | null;
|
|
7
|
+
export declare function resolveHfBaseUrl(modelId: string, revision?: string): string | null;
|