doppler-gpu 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +19 -15
- package/package.json +77 -29
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +64 -6
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +524 -350
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +12 -2
- package/src/config/schema/doppler.schema.js +108 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +68 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +76 -0
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +107 -0
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +3 -21
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +6 -5
- package/src/gpu/kernels/index.js +3 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +12 -0
- package/src/inference/browser-harness-text-helpers.js +223 -14
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +264 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +22 -27
- package/src/inference/pipelines/text/embed.js +0 -39
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +4 -5
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +8 -3
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +223 -547
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +5 -2
- package/src/inference/pipelines/text/init.js +183 -724
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.js +2 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +120 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +223 -27
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +13 -3
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +11 -3
- package/src/storage/artifact-storage-context.js +231 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +21 -24
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +2 -0
- package/src/tooling/source-runtime-bundle.js +3 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
|
|
2
2
|
import { getDevice, initDevice, getKernelCapabilities } from '../../gpu/device.js';
|
|
3
|
-
import { getBufferPool as getGlobalBufferPool, releaseBuffer } from '../../memory/buffer-pool.js';
|
|
3
|
+
import { getBufferPool as getGlobalBufferPool, readBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
|
|
4
4
|
import { log } from '../../debug/index.js';
|
|
5
5
|
import { configurePerfGuards } from '../../gpu/perf-guards.js';
|
|
6
6
|
import { MoERouter } from '../moe-router.js';
|
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
applyModelBatchingRuntimeDefaults,
|
|
30
30
|
resolveKernelPathState,
|
|
31
31
|
initTokenizerFromManifest,
|
|
32
|
+
assertManifestComputeLaneBinding,
|
|
32
33
|
} from './text/model-load.js';
|
|
33
34
|
import { resolvePerLayerInputsSession } from './text/generator-helpers.js';
|
|
34
35
|
import { getKernelPathActivationDtype } from '../../config/kernel-path-loader.js';
|
|
@@ -47,6 +48,31 @@ import {
|
|
|
47
48
|
import { getDopplerLoader } from '../../loader/doppler-loader.js';
|
|
48
49
|
import { registerPipeline, getPipelineFactory } from './registry.js';
|
|
49
50
|
import { selectRuleValue } from '../../rules/rule-registry.js';
|
|
51
|
+
|
|
52
|
+
// AbortSignal contract: every public inference primitive on this pipeline
|
|
53
|
+
// accepts `options.signal` (or `args.signal`). When the signal aborts the
|
|
54
|
+
// call rejects with `AbortError` so callers (the dream typing-clock
|
|
55
|
+
// controller, batch eval scripts, etc.) can cancel superseded work without
|
|
56
|
+
// waiting for the model to finish. Internal checkpoints fire between
|
|
57
|
+
// dispatches; mid-dispatch cancellation is not provided because WebGPU
|
|
58
|
+
// command-buffer cancellation is not exposed by the spec.
|
|
59
|
+
export class AbortError extends Error {
|
|
60
|
+
constructor(message = 'Doppler: aborted') {
|
|
61
|
+
super(message);
|
|
62
|
+
this.name = 'AbortError';
|
|
63
|
+
this.code = 'ABORT_ERR';
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function isAbortError(err) {
|
|
68
|
+
return !!err && (err.name === 'AbortError' || err.code === 'ABORT_ERR' || err.code === 20);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function assertNotAborted(signal) {
|
|
72
|
+
if (signal && signal.aborted) {
|
|
73
|
+
throw new AbortError(typeof signal.reason === 'string' ? signal.reason : 'Doppler: aborted');
|
|
74
|
+
}
|
|
75
|
+
}
|
|
50
76
|
import { initConvLayerState } from './text/ops.js';
|
|
51
77
|
import { destroyPleBufferCache, destroyPleRuntimeCache } from './text/per-layer-inputs.js';
|
|
52
78
|
|
|
@@ -56,6 +82,35 @@ function destroyMoERouter(router) {
|
|
|
56
82
|
}
|
|
57
83
|
}
|
|
58
84
|
|
|
85
|
+
function createPipelineLoadPhaseError(error, phase, context = {}) {
|
|
86
|
+
const message = error?.message || String(error);
|
|
87
|
+
const wrapped = new Error(
|
|
88
|
+
`Pipeline load phase "${phase}" failed: ${message}`,
|
|
89
|
+
error instanceof Error ? { cause: error } : undefined
|
|
90
|
+
);
|
|
91
|
+
wrapped.name = error?.name || 'Error';
|
|
92
|
+
if (error?.code !== undefined) {
|
|
93
|
+
wrapped.code = error.code;
|
|
94
|
+
}
|
|
95
|
+
wrapped.details = {
|
|
96
|
+
...(error?.details && typeof error.details === 'object' ? error.details : {}),
|
|
97
|
+
pipelineLoadPhase: phase,
|
|
98
|
+
...context,
|
|
99
|
+
};
|
|
100
|
+
return wrapped;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async function withPipelineLoadPhase(phase, context, run) {
|
|
104
|
+
try {
|
|
105
|
+
return await run();
|
|
106
|
+
} catch (error) {
|
|
107
|
+
if (error?.details?.pipelineLoadPhase) {
|
|
108
|
+
throw error;
|
|
109
|
+
}
|
|
110
|
+
throw createPipelineLoadPhaseError(error, phase, context);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
59
114
|
function resolveSingleSpecialTokenId(tokenizer, tokenText, label) {
|
|
60
115
|
const rawTokenIds = tokenizer?.encode?.(tokenText);
|
|
61
116
|
const tokenIds = Array.isArray(rawTokenIds)
|
|
@@ -287,12 +342,17 @@ export class InferencePipeline extends PipelineState {
|
|
|
287
342
|
const fallbackInfo = this.executionV1State.fallbackKernelPath
|
|
288
343
|
? ', fallbackKernelPath=yes'
|
|
289
344
|
: '';
|
|
345
|
+
const laneIntegrity = this.executionV1State.laneIntegrity;
|
|
346
|
+
const laneInfo = laneIntegrity?.status === 'transformed'
|
|
347
|
+
? `, laneIntegrity=transformed(declared=${laneIntegrity.declared.activationDtype}/${laneIntegrity.declared.kvDtype},` +
|
|
348
|
+
`executed=${laneIntegrity.executed.activationDtype}/${laneIntegrity.executed.kvDtype})`
|
|
349
|
+
: '';
|
|
290
350
|
log.info(
|
|
291
351
|
'Pipeline',
|
|
292
352
|
`Execution v1 enabled (steps=${this.executionV1State.resolvedSteps.all.length}, ` +
|
|
293
353
|
`kernelPathInline=${this.executionV1State.runtimeInferencePatch.kernelPath ? 'yes' : 'no'}, ` +
|
|
294
354
|
`pipelineInline=${this.executionV1State.runtimeInferencePatch.pipeline ? 'yes' : 'no'}` +
|
|
295
|
-
`${transformInfo}${fallbackInfo})`
|
|
355
|
+
`${transformInfo}${laneInfo}${fallbackInfo})`
|
|
296
356
|
);
|
|
297
357
|
}
|
|
298
358
|
}
|
|
@@ -378,10 +438,14 @@ export class InferencePipeline extends PipelineState {
|
|
|
378
438
|
const kernelInfo = this.resolvedKernelPath ? `kernelPath=${this.resolvedKernelPath.id}` : 'kernelPath=none';
|
|
379
439
|
log.info('Pipeline', `${cfg.numLayers}L/${cfg.hiddenSize}H/${cfg.numHeads}heads (${cfg.headDim}dim)${moeStr}, ${kernelInfo}`);
|
|
380
440
|
|
|
381
|
-
this.tokenizer = await
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
441
|
+
this.tokenizer = await withPipelineLoadPhase(
|
|
442
|
+
'tokenizer',
|
|
443
|
+
{ modelId: manifest.modelId ?? null },
|
|
444
|
+
() => initTokenizerFromManifest(
|
|
445
|
+
manifest,
|
|
446
|
+
this.baseUrl,
|
|
447
|
+
this.storageContext
|
|
448
|
+
)
|
|
385
449
|
);
|
|
386
450
|
const tokenizerVocabSize = this.tokenizer.getVocabSize();
|
|
387
451
|
if (Number.isFinite(tokenizerVocabSize) && tokenizerVocabSize > 0) {
|
|
@@ -390,23 +454,8 @@ export class InferencePipeline extends PipelineState {
|
|
|
390
454
|
}
|
|
391
455
|
}
|
|
392
456
|
|
|
393
|
-
//
|
|
394
|
-
|
|
395
|
-
const manifestComputeHint = manifest?.quantizationInfo?.compute;
|
|
396
|
-
const resolvedKvDtype = this.runtimeConfig.inference.session?.kvcache?.kvDtype;
|
|
397
|
-
if (
|
|
398
|
-
manifestComputeHint
|
|
399
|
-
&& resolvedKvDtype
|
|
400
|
-
&& String(manifestComputeHint).toLowerCase() !== String(resolvedKvDtype).toLowerCase()
|
|
401
|
-
) {
|
|
402
|
-
log.warn(
|
|
403
|
-
'Pipeline',
|
|
404
|
-
`KV cache kvDtype from execution-v1 resolution (${resolvedKvDtype}) differs from ` +
|
|
405
|
-
`manifest quantizationInfo.compute hint (${manifestComputeHint}). ` +
|
|
406
|
-
`The kernel path dtype contract takes precedence.`
|
|
407
|
-
);
|
|
408
|
-
}
|
|
409
|
-
}
|
|
457
|
+
// Manifest quantizationInfo.compute is the binding lane identity.
|
|
458
|
+
assertManifestComputeLaneBinding({ manifest, runtimeConfig: this.runtimeConfig });
|
|
410
459
|
|
|
411
460
|
// Initialize KV cache
|
|
412
461
|
if (this.modelConfig.decodeStrategy === 'replay_prefill') {
|
|
@@ -465,7 +514,11 @@ export class InferencePipeline extends PipelineState {
|
|
|
465
514
|
}
|
|
466
515
|
|
|
467
516
|
// Load weights
|
|
468
|
-
await
|
|
517
|
+
await withPipelineLoadPhase(
|
|
518
|
+
'loadWeights',
|
|
519
|
+
{ modelId: manifest.modelId ?? null },
|
|
520
|
+
() => this._loadWeights()
|
|
521
|
+
);
|
|
469
522
|
|
|
470
523
|
// Initialize RoPE frequencies
|
|
471
524
|
await this._initRoPE();
|
|
@@ -920,13 +973,15 @@ export class InferencePipeline extends PipelineState {
|
|
|
920
973
|
* @param {number} [params.softTokenBudget] Per-request soft token budget (Gemma 4 tiers: 70/140/280/560/1120)
|
|
921
974
|
* @returns {Promise<{ text: string, tokens: number[] }>}
|
|
922
975
|
*/
|
|
923
|
-
async transcribeImage({ imageBytes, width, height, prompt, maxTokens, softTokenBudget }) {
|
|
976
|
+
async transcribeImage({ imageBytes, width, height, prompt, maxTokens, softTokenBudget, signal }) {
|
|
977
|
+
assertNotAborted(signal);
|
|
924
978
|
if (!this.visionCapable) {
|
|
925
979
|
throw new Error(
|
|
926
980
|
'Pipeline does not support image transcription (no image_token_id in manifest).'
|
|
927
981
|
);
|
|
928
982
|
}
|
|
929
983
|
await this._ensureVisionWeightsLoaded();
|
|
984
|
+
assertNotAborted(signal);
|
|
930
985
|
|
|
931
986
|
this.reset();
|
|
932
987
|
|
|
@@ -1061,7 +1116,8 @@ export class InferencePipeline extends PipelineState {
|
|
|
1061
1116
|
* @param {number} [params.perFrameSoftTokenBudget] Soft token budget per frame
|
|
1062
1117
|
* @returns {Promise<{ text: string, tokens: number[] }>}
|
|
1063
1118
|
*/
|
|
1064
|
-
async transcribeVideo({ frames, prompt, maxTokens, maxFrames, perFrameSoftTokenBudget }) {
|
|
1119
|
+
async transcribeVideo({ frames, prompt, maxTokens, maxFrames, perFrameSoftTokenBudget, signal }) {
|
|
1120
|
+
assertNotAborted(signal);
|
|
1065
1121
|
if (!this.visionCapable) {
|
|
1066
1122
|
throw new Error(
|
|
1067
1123
|
'Pipeline does not support video transcription (no image_token_id in manifest for vision encoder).'
|
|
@@ -1184,7 +1240,8 @@ export class InferencePipeline extends PipelineState {
|
|
|
1184
1240
|
* @param {number} [params.maxTokens] Max tokens to generate
|
|
1185
1241
|
* @returns {Promise<{ text: string, tokens: number[] }>}
|
|
1186
1242
|
*/
|
|
1187
|
-
async transcribeAudio({ audio, prompt, maxTokens }) {
|
|
1243
|
+
async transcribeAudio({ audio, prompt, maxTokens, signal }) {
|
|
1244
|
+
assertNotAborted(signal);
|
|
1188
1245
|
if (!this.audioCapable) {
|
|
1189
1246
|
throw new Error(
|
|
1190
1247
|
'Pipeline does not support audio transcription (no audio_token_id in manifest).'
|
|
@@ -1353,40 +1410,50 @@ export class InferencePipeline extends PipelineState {
|
|
|
1353
1410
|
|
|
1354
1411
|
|
|
1355
1412
|
generate(prompt, options = {}) {
|
|
1413
|
+
assertNotAborted(options?.signal);
|
|
1356
1414
|
return this.generator.generate(prompt, options);
|
|
1357
1415
|
}
|
|
1358
1416
|
|
|
1359
1417
|
generateTokens(prompt, options = {}) {
|
|
1418
|
+
assertNotAborted(options?.signal);
|
|
1360
1419
|
return this.generator.generateTokens(prompt, options);
|
|
1361
1420
|
}
|
|
1362
1421
|
|
|
1363
1422
|
generateTokenIds(prompt, options = {}) {
|
|
1423
|
+
assertNotAborted(options?.signal);
|
|
1364
1424
|
return this.generator.generateTokenIds(prompt, options);
|
|
1365
1425
|
}
|
|
1366
1426
|
|
|
1367
1427
|
decodeStepLogits(currentIds, options = {}) {
|
|
1428
|
+
assertNotAborted(options?.signal);
|
|
1368
1429
|
return this.generator.decodeStepLogits(currentIds, options);
|
|
1369
1430
|
}
|
|
1370
1431
|
|
|
1371
1432
|
advanceWithToken(tokenId, options = {}) {
|
|
1433
|
+
assertNotAborted(options?.signal);
|
|
1372
1434
|
return this.generator.advanceWithToken(tokenId, options);
|
|
1373
1435
|
}
|
|
1374
1436
|
|
|
1375
1437
|
advanceWithTokenAndEmbedding(tokenId, options = {}) {
|
|
1438
|
+
assertNotAborted(options?.signal);
|
|
1376
1439
|
return this.generator.advanceWithTokenAndEmbedding(tokenId, options);
|
|
1377
1440
|
}
|
|
1378
1441
|
|
|
1379
1442
|
|
|
1380
1443
|
prefillKVOnly(prompt, options = {}) {
|
|
1444
|
+
assertNotAborted(options?.signal);
|
|
1381
1445
|
return this.generator.prefillKVOnly(prompt, options);
|
|
1382
1446
|
}
|
|
1383
1447
|
|
|
1384
1448
|
prefillWithEmbedding(prompt, options = {}) {
|
|
1449
|
+
assertNotAborted(options?.signal);
|
|
1385
1450
|
return this.generator.prefillWithEmbedding(prompt, options);
|
|
1386
1451
|
}
|
|
1387
1452
|
|
|
1388
1453
|
async embed(prompt, options = {}) {
|
|
1454
|
+
assertNotAborted(options?.signal);
|
|
1389
1455
|
const result = await this.prefillWithEmbedding(prompt, options);
|
|
1456
|
+
assertNotAborted(options?.signal);
|
|
1390
1457
|
return {
|
|
1391
1458
|
embedding: result.embedding,
|
|
1392
1459
|
tokens: result.tokens,
|
|
@@ -1399,15 +1466,144 @@ export class InferencePipeline extends PipelineState {
|
|
|
1399
1466
|
if (!Array.isArray(prompts)) {
|
|
1400
1467
|
throw new Error('embedBatch expects an array of prompts');
|
|
1401
1468
|
}
|
|
1469
|
+
assertNotAborted(options?.signal);
|
|
1402
1470
|
const batchOptions = { ...options, __skipStateSnapshot: true };
|
|
1403
1471
|
const outputs = [];
|
|
1404
1472
|
for (const prompt of prompts) {
|
|
1473
|
+
// Check between every prompt so a superseded revision drops the rest.
|
|
1474
|
+
assertNotAborted(options?.signal);
|
|
1405
1475
|
outputs.push(await this.embed(prompt, batchOptions));
|
|
1406
1476
|
this.resetForBatch();
|
|
1407
1477
|
}
|
|
1408
1478
|
return outputs;
|
|
1409
1479
|
}
|
|
1410
1480
|
|
|
1481
|
+
// Run the vision encoder over a single image and return a mean-pooled
|
|
1482
|
+
// embedding in the model's text-hidden-size space. Decoder responsibility
|
|
1483
|
+
// (jpeg/png -> RGBA pixels) belongs to the caller; this method takes
|
|
1484
|
+
// already-decoded pixel data.
|
|
1485
|
+
async embedImage({ pixels, width, height, softTokenBudget, signal } = {}) {
|
|
1486
|
+
assertNotAborted(signal);
|
|
1487
|
+
if (!this.visionCapable) {
|
|
1488
|
+
throw new Error(
|
|
1489
|
+
'Pipeline does not support image embedding (no image_token_id in manifest).'
|
|
1490
|
+
);
|
|
1491
|
+
}
|
|
1492
|
+
if (pixels == null) {
|
|
1493
|
+
throw new Error('[Pipeline] embedImage: pixels are required.');
|
|
1494
|
+
}
|
|
1495
|
+
if (!Number.isFinite(width) || width <= 0 || !Number.isFinite(height) || height <= 0) {
|
|
1496
|
+
throw new Error('[Pipeline] embedImage: width and height must be positive integers.');
|
|
1497
|
+
}
|
|
1498
|
+
await this._ensureVisionWeightsLoaded();
|
|
1499
|
+
this.reset();
|
|
1500
|
+
|
|
1501
|
+
const { encodeImage } = await import('./vision/index.js');
|
|
1502
|
+
const encodeResult = await encodeImage({
|
|
1503
|
+
pixels,
|
|
1504
|
+
width,
|
|
1505
|
+
height,
|
|
1506
|
+
visionConfig: this.visionConfig,
|
|
1507
|
+
weights: this.visionWeights,
|
|
1508
|
+
softTokenBudget,
|
|
1509
|
+
});
|
|
1510
|
+
|
|
1511
|
+
const hiddenSize = this.modelConfig.hiddenSize;
|
|
1512
|
+
const numTokens = encodeResult.numTokens;
|
|
1513
|
+
if (!Number.isFinite(numTokens) || numTokens < 1) {
|
|
1514
|
+
releaseBuffer(encodeResult.features);
|
|
1515
|
+
throw new Error(`[Pipeline] embedImage: encoder produced ${numTokens} soft tokens; expected >= 1.`);
|
|
1516
|
+
}
|
|
1517
|
+
try {
|
|
1518
|
+
const bytes = await readBuffer(
|
|
1519
|
+
encodeResult.features,
|
|
1520
|
+
numTokens * hiddenSize * Float32Array.BYTES_PER_ELEMENT
|
|
1521
|
+
);
|
|
1522
|
+
const features = new Float32Array(bytes);
|
|
1523
|
+
const pooled = new Float32Array(hiddenSize);
|
|
1524
|
+
for (let t = 0; t < numTokens; t++) {
|
|
1525
|
+
const base = t * hiddenSize;
|
|
1526
|
+
for (let d = 0; d < hiddenSize; d++) {
|
|
1527
|
+
pooled[d] += features[base + d];
|
|
1528
|
+
}
|
|
1529
|
+
}
|
|
1530
|
+
const inv = 1 / numTokens;
|
|
1531
|
+
for (let d = 0; d < hiddenSize; d++) pooled[d] *= inv;
|
|
1532
|
+
return {
|
|
1533
|
+
embedding: pooled,
|
|
1534
|
+
embeddingDim: hiddenSize,
|
|
1535
|
+
numTokens,
|
|
1536
|
+
embeddingMode: 'mean',
|
|
1537
|
+
};
|
|
1538
|
+
} finally {
|
|
1539
|
+
releaseBuffer(encodeResult.features);
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
|
|
1543
|
+
// Run the audio encoder over a single PCM segment and return a mean-pooled
|
|
1544
|
+
// embedding in the model's audio-projection-output space (which equals the
|
|
1545
|
+
// text hidden size in Gemma 4). Decoder responsibility (webm/opus/wav ->
|
|
1546
|
+
// Float32 PCM at the model's expected sample rate) belongs to the caller.
|
|
1547
|
+
async embedAudio({ audio, signal } = {}) {
|
|
1548
|
+
assertNotAborted(signal);
|
|
1549
|
+
if (!this.audioCapable) {
|
|
1550
|
+
throw new Error(
|
|
1551
|
+
'Pipeline does not support audio embedding (no audio_token_id in manifest).'
|
|
1552
|
+
);
|
|
1553
|
+
}
|
|
1554
|
+
if (audio == null) {
|
|
1555
|
+
throw new Error('[Pipeline] embedAudio: audio is required.');
|
|
1556
|
+
}
|
|
1557
|
+
await this._ensureAudioWeightsLoaded();
|
|
1558
|
+
this.reset();
|
|
1559
|
+
|
|
1560
|
+
const { extractLogMelSpectrogram } = await import('./audio/mel.js');
|
|
1561
|
+
const { encodeAudio } = await import('./audio/index.js');
|
|
1562
|
+
const { features: melFeatures, numFrames, nMels } = extractLogMelSpectrogram(audio);
|
|
1563
|
+
const encodeResult = await encodeAudio({
|
|
1564
|
+
melFeatures,
|
|
1565
|
+
numFrames,
|
|
1566
|
+
nMels,
|
|
1567
|
+
audioConfig: this.audioConfig,
|
|
1568
|
+
weights: this.audioWeights,
|
|
1569
|
+
});
|
|
1570
|
+
|
|
1571
|
+
const hiddenSize = Number(this.audioConfig?.outputProjDims ?? this.modelConfig?.hiddenSize);
|
|
1572
|
+
const numTokens = encodeResult.numTokens;
|
|
1573
|
+
if (!Number.isFinite(hiddenSize) || hiddenSize < 1) {
|
|
1574
|
+
releaseBuffer(encodeResult.features);
|
|
1575
|
+
throw new Error('[Pipeline] embedAudio: audioConfig.outputProjDims is missing or invalid.');
|
|
1576
|
+
}
|
|
1577
|
+
if (!Number.isFinite(numTokens) || numTokens < 1) {
|
|
1578
|
+
releaseBuffer(encodeResult.features);
|
|
1579
|
+
throw new Error(`[Pipeline] embedAudio: encoder produced ${numTokens} tokens; expected >= 1.`);
|
|
1580
|
+
}
|
|
1581
|
+
try {
|
|
1582
|
+
const bytes = await readBuffer(
|
|
1583
|
+
encodeResult.features,
|
|
1584
|
+
numTokens * hiddenSize * Float32Array.BYTES_PER_ELEMENT
|
|
1585
|
+
);
|
|
1586
|
+
const features = new Float32Array(bytes);
|
|
1587
|
+
const pooled = new Float32Array(hiddenSize);
|
|
1588
|
+
for (let t = 0; t < numTokens; t++) {
|
|
1589
|
+
const base = t * hiddenSize;
|
|
1590
|
+
for (let d = 0; d < hiddenSize; d++) {
|
|
1591
|
+
pooled[d] += features[base + d];
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
const inv = 1 / numTokens;
|
|
1595
|
+
for (let d = 0; d < hiddenSize; d++) pooled[d] *= inv;
|
|
1596
|
+
return {
|
|
1597
|
+
embedding: pooled,
|
|
1598
|
+
embeddingDim: hiddenSize,
|
|
1599
|
+
numTokens,
|
|
1600
|
+
embeddingMode: 'mean',
|
|
1601
|
+
};
|
|
1602
|
+
} finally {
|
|
1603
|
+
releaseBuffer(encodeResult.features);
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
|
|
1411
1607
|
prefillWithLogits(prompt, options = {}) {
|
|
1412
1608
|
return this.generator.prefillWithLogits(prompt, options);
|
|
1413
1609
|
}
|
|
@@ -81,113 +81,6 @@ export async function encodeImage(params) {
|
|
|
81
81
|
};
|
|
82
82
|
}
|
|
83
83
|
|
|
84
|
-
/**
|
|
85
|
-
* Inject visual tokens into text token embeddings.
|
|
86
|
-
*
|
|
87
|
-
* Replaces positions in the embedding sequence where image_token_id appears
|
|
88
|
-
* with the encoded visual features from the vision encoder.
|
|
89
|
-
*
|
|
90
|
-
* For Qwen3-VL with DeepStack, visual tokens are injected at specific decoder
|
|
91
|
-
* layers (deepstackVisualIndexes), not at the input embedding level.
|
|
92
|
-
* This function handles the simpler input-level injection case.
|
|
93
|
-
* DeepStack injection is handled in the decoder layer loop.
|
|
94
|
-
*
|
|
95
|
-
* @param {object} params
|
|
96
|
-
* @param {Float32Array} params.textEmbeddings [seqLen, hiddenSize]
|
|
97
|
-
* @param {Int32Array} params.tokenIds [seqLen]
|
|
98
|
-
* @param {GPUBuffer} params.visualFeatures [numVisualTokens, outHiddenSize]
|
|
99
|
-
* @param {number} params.numVisualTokens Number of visual tokens
|
|
100
|
-
* @param {number} params.imageTokenId Token ID marking image positions
|
|
101
|
-
* @param {number} params.hiddenSize Text model hidden size
|
|
102
|
-
* @returns {{ mergedEmbeddings: Float32Array, mergedLength: number }}
|
|
103
|
-
*/
|
|
104
|
-
export function mergeVisualTokens(params) {
|
|
105
|
-
const {
|
|
106
|
-
textEmbeddings, tokenIds, visualFeatures,
|
|
107
|
-
numVisualTokens, imageTokenId, hiddenSize,
|
|
108
|
-
} = params;
|
|
109
|
-
|
|
110
|
-
// Count image token positions.
|
|
111
|
-
const imagePositions = [];
|
|
112
|
-
for (let i = 0; i < tokenIds.length; i++) {
|
|
113
|
-
if (tokenIds[i] === imageTokenId) {
|
|
114
|
-
imagePositions.push(i);
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
if (imagePositions.length === 0) {
|
|
119
|
-
log.debug('Vision', 'mergeVisualTokens: no image tokens found, returning text-only');
|
|
120
|
-
return { mergedEmbeddings: textEmbeddings, mergedLength: tokenIds.length };
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
log.debug('Vision', `mergeVisualTokens: replacing ${imagePositions.length} image tokens with ${numVisualTokens} visual tokens`);
|
|
124
|
-
|
|
125
|
-
// The merged sequence replaces contiguous image_token_id runs with visual features.
|
|
126
|
-
// For Qwen3-VL: image tokens appear as a block between vision_start and vision_end tokens.
|
|
127
|
-
// The visual features replace the entire image token block.
|
|
128
|
-
|
|
129
|
-
// Find contiguous image token ranges.
|
|
130
|
-
const ranges = [];
|
|
131
|
-
let rangeStart = imagePositions[0];
|
|
132
|
-
let rangeEnd = imagePositions[0];
|
|
133
|
-
for (let i = 1; i < imagePositions.length; i++) {
|
|
134
|
-
if (imagePositions[i] === rangeEnd + 1) {
|
|
135
|
-
rangeEnd = imagePositions[i];
|
|
136
|
-
} else {
|
|
137
|
-
ranges.push([rangeStart, rangeEnd]);
|
|
138
|
-
rangeStart = imagePositions[i];
|
|
139
|
-
rangeEnd = imagePositions[i];
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
ranges.push([rangeStart, rangeEnd]);
|
|
143
|
-
|
|
144
|
-
// Build merged sequence: text tokens (non-image) + visual tokens replacing each range.
|
|
145
|
-
const textLen = tokenIds.length;
|
|
146
|
-
const replacedCount = imagePositions.length;
|
|
147
|
-
const mergedLength = textLen - replacedCount + numVisualTokens;
|
|
148
|
-
const merged = new Float32Array(mergedLength * hiddenSize);
|
|
149
|
-
|
|
150
|
-
let srcPos = 0;
|
|
151
|
-
let dstPos = 0;
|
|
152
|
-
let visualOffset = 0;
|
|
153
|
-
|
|
154
|
-
for (const [start, end] of ranges) {
|
|
155
|
-
// Copy text tokens before this range.
|
|
156
|
-
const textBefore = start - srcPos;
|
|
157
|
-
if (textBefore > 0) {
|
|
158
|
-
merged.set(
|
|
159
|
-
textEmbeddings.subarray(srcPos * hiddenSize, start * hiddenSize),
|
|
160
|
-
dstPos * hiddenSize,
|
|
161
|
-
);
|
|
162
|
-
dstPos += textBefore;
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
// Insert visual tokens replacing this range.
|
|
166
|
-
const rangeLen = end - start + 1;
|
|
167
|
-
const tokensToInsert = Math.min(numVisualTokens - visualOffset, rangeLen);
|
|
168
|
-
// Copy from visual features buffer (CPU side).
|
|
169
|
-
for (let i = 0; i < tokensToInsert; i++) {
|
|
170
|
-
for (let d = 0; d < hiddenSize; d++) {
|
|
171
|
-
merged[(dstPos + i) * hiddenSize + d] = visualFeatures[(visualOffset + i) * hiddenSize + d];
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
dstPos += tokensToInsert;
|
|
175
|
-
visualOffset += tokensToInsert;
|
|
176
|
-
srcPos = end + 1;
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
// Copy remaining text tokens after last range.
|
|
180
|
-
if (srcPos < textLen) {
|
|
181
|
-
merged.set(
|
|
182
|
-
textEmbeddings.subarray(srcPos * hiddenSize, textLen * hiddenSize),
|
|
183
|
-
dstPos * hiddenSize,
|
|
184
|
-
);
|
|
185
|
-
dstPos += textLen - srcPos;
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
return { mergedEmbeddings: merged, mergedLength: dstPos };
|
|
189
|
-
}
|
|
190
|
-
|
|
191
84
|
/**
|
|
192
85
|
* @typedef {object} VisionEncodeResult
|
|
193
86
|
* @property {GPUBuffer} features Encoded visual tokens [numTokens, outHiddenSize]
|
|
@@ -80,10 +80,3 @@ export interface CreateRuntimeModelContractOptions
|
|
|
80
80
|
export declare function createRuntimeModelContract(
|
|
81
81
|
options: CreateRuntimeModelContractOptions
|
|
82
82
|
): RuntimeModelContract;
|
|
83
|
-
|
|
84
|
-
export declare function createRuntimeModelFromManifest<T extends Record<string, unknown>>(
|
|
85
|
-
manifest: T,
|
|
86
|
-
options?: { sourceFormat?: string | null }
|
|
87
|
-
): T & RuntimeModelContract;
|
|
88
|
-
|
|
89
|
-
export declare function isRuntimeModelContract(value: unknown): value is RuntimeModelContract;
|
|
@@ -127,26 +127,3 @@ export function createRuntimeModelContract(options = {}) {
|
|
|
127
127
|
};
|
|
128
128
|
}
|
|
129
129
|
|
|
130
|
-
export function createRuntimeModelFromManifest(manifest, options = {}) {
|
|
131
|
-
if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) {
|
|
132
|
-
throw new Error('createRuntimeModelFromManifest requires a manifest-like object.');
|
|
133
|
-
}
|
|
134
|
-
return {
|
|
135
|
-
...manifest,
|
|
136
|
-
kind: 'runtime-model',
|
|
137
|
-
sourceFormat: normalizeText(options.sourceFormat).toLowerCase()
|
|
138
|
-
|| normalizeText(manifest?.sourceFormat).toLowerCase()
|
|
139
|
-
|| 'rdrr',
|
|
140
|
-
};
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
export function isRuntimeModelContract(value) {
|
|
144
|
-
return value != null
|
|
145
|
-
&& typeof value === 'object'
|
|
146
|
-
&& !Array.isArray(value)
|
|
147
|
-
&& Array.isArray(value.shards)
|
|
148
|
-
&& typeof value.modelId === 'string'
|
|
149
|
-
&& value.modelId.trim().length > 0
|
|
150
|
-
&& value.inference != null
|
|
151
|
-
&& typeof value.inference === 'object';
|
|
152
|
-
}
|
|
@@ -53,6 +53,8 @@ export interface InferenceHarnessOptions {
|
|
|
53
53
|
baseUrl?: string;
|
|
54
54
|
/** Runtime overrides for kernel selection */
|
|
55
55
|
runtime?: RuntimeOverrides;
|
|
56
|
+
/** Explicit source loading mode from the command/suite request */
|
|
57
|
+
loadMode?: 'opfs' | 'http' | 'memory' | 'file' | null;
|
|
56
58
|
/** Progress callback */
|
|
57
59
|
onProgress?: (phase: string, progress: number, detail?: string) => void;
|
|
58
60
|
/** Log function (default: debug log) */
|
|
@@ -112,37 +114,19 @@ export declare function parseRuntimeOverridesFromURL(
|
|
|
112
114
|
* @param log - Optional logging function
|
|
113
115
|
* @returns Async function that loads a shard by index
|
|
114
116
|
*/
|
|
115
|
-
export declare function createHttpShardLoader(
|
|
116
|
-
baseUrl: string,
|
|
117
|
-
manifest: RDRRManifest,
|
|
118
|
-
log?: (msg: string, level?: string) => void
|
|
119
|
-
): (idx: number) => Promise<Uint8Array>;
|
|
120
|
-
|
|
121
117
|
export declare function createHarnessShardStorageContext(
|
|
122
118
|
modelUrl: string,
|
|
123
119
|
manifest: RDRRManifest,
|
|
124
|
-
log?: (msg: string, level?: string) => void
|
|
120
|
+
log?: (msg: string, level?: string) => void,
|
|
121
|
+
options?: {
|
|
122
|
+
loadMode?: 'opfs' | 'http' | 'memory' | 'file' | null;
|
|
123
|
+
}
|
|
125
124
|
): PipelineStorageContext;
|
|
126
125
|
|
|
127
126
|
// ============================================================================
|
|
128
127
|
// Pipeline Initialization
|
|
129
128
|
// ============================================================================
|
|
130
129
|
|
|
131
|
-
/**
|
|
132
|
-
* Fetch and parse a model manifest from a URL.
|
|
133
|
-
*
|
|
134
|
-
* @param manifestUrl - URL to manifest.json
|
|
135
|
-
* @returns Parsed manifest
|
|
136
|
-
*/
|
|
137
|
-
export declare function fetchManifest(manifestUrl: string): Promise<RDRRManifest>;
|
|
138
|
-
|
|
139
|
-
/**
|
|
140
|
-
* Initialize the WebGPU device and return capabilities.
|
|
141
|
-
*
|
|
142
|
-
* @returns Kernel capabilities
|
|
143
|
-
*/
|
|
144
|
-
export declare function initializeDevice(): Promise<KernelCapabilities>;
|
|
145
|
-
|
|
146
130
|
/**
|
|
147
131
|
* Initialize a complete inference pipeline from a model URL.
|
|
148
132
|
*
|