doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
|
|
2
2
|
import { getDevice, initDevice, getKernelCapabilities } from '../../gpu/device.js';
|
|
3
|
-
import { getBufferPool as getGlobalBufferPool, releaseBuffer } from '../../memory/buffer-pool.js';
|
|
3
|
+
import { getBufferPool as getGlobalBufferPool, readBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
|
|
4
4
|
import { log } from '../../debug/index.js';
|
|
5
5
|
import { configurePerfGuards } from '../../gpu/perf-guards.js';
|
|
6
6
|
import { MoERouter } from '../moe-router.js';
|
|
@@ -29,7 +29,9 @@ import {
|
|
|
29
29
|
applyModelBatchingRuntimeDefaults,
|
|
30
30
|
resolveKernelPathState,
|
|
31
31
|
initTokenizerFromManifest,
|
|
32
|
+
assertManifestComputeLaneBinding,
|
|
32
33
|
} from './text/model-load.js';
|
|
34
|
+
import { resolvePerLayerInputsSession } from './text/generator-helpers.js';
|
|
33
35
|
import { getKernelPathActivationDtype } from '../../config/kernel-path-loader.js';
|
|
34
36
|
import { applyPipelineDebugConfig } from './text/debug-utils.js';
|
|
35
37
|
import { resolveLayerPipeline } from './text/layer-plan.js';
|
|
@@ -46,6 +48,31 @@ import {
|
|
|
46
48
|
import { getDopplerLoader } from '../../loader/doppler-loader.js';
|
|
47
49
|
import { registerPipeline, getPipelineFactory } from './registry.js';
|
|
48
50
|
import { selectRuleValue } from '../../rules/rule-registry.js';
|
|
51
|
+
|
|
52
|
+
// AbortSignal contract: every public inference primitive on this pipeline
|
|
53
|
+
// accepts `options.signal` (or `args.signal`). When the signal aborts the
|
|
54
|
+
// call rejects with `AbortError` so callers (the dream typing-clock
|
|
55
|
+
// controller, batch eval scripts, etc.) can cancel superseded work without
|
|
56
|
+
// waiting for the model to finish. Internal checkpoints fire between
|
|
57
|
+
// dispatches; mid-dispatch cancellation is not provided because WebGPU
|
|
58
|
+
// command-buffer cancellation is not exposed by the spec.
|
|
59
|
+
export class AbortError extends Error {
|
|
60
|
+
constructor(message = 'Doppler: aborted') {
|
|
61
|
+
super(message);
|
|
62
|
+
this.name = 'AbortError';
|
|
63
|
+
this.code = 'ABORT_ERR';
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function isAbortError(err) {
|
|
68
|
+
return !!err && (err.name === 'AbortError' || err.code === 'ABORT_ERR' || err.code === 20);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function assertNotAborted(signal) {
|
|
72
|
+
if (signal && signal.aborted) {
|
|
73
|
+
throw new AbortError(typeof signal.reason === 'string' ? signal.reason : 'Doppler: aborted');
|
|
74
|
+
}
|
|
75
|
+
}
|
|
49
76
|
import { initConvLayerState } from './text/ops.js';
|
|
50
77
|
import { destroyPleBufferCache, destroyPleRuntimeCache } from './text/per-layer-inputs.js';
|
|
51
78
|
|
|
@@ -55,6 +82,35 @@ function destroyMoERouter(router) {
|
|
|
55
82
|
}
|
|
56
83
|
}
|
|
57
84
|
|
|
85
|
+
function createPipelineLoadPhaseError(error, phase, context = {}) {
|
|
86
|
+
const message = error?.message || String(error);
|
|
87
|
+
const wrapped = new Error(
|
|
88
|
+
`Pipeline load phase "${phase}" failed: ${message}`,
|
|
89
|
+
error instanceof Error ? { cause: error } : undefined
|
|
90
|
+
);
|
|
91
|
+
wrapped.name = error?.name || 'Error';
|
|
92
|
+
if (error?.code !== undefined) {
|
|
93
|
+
wrapped.code = error.code;
|
|
94
|
+
}
|
|
95
|
+
wrapped.details = {
|
|
96
|
+
...(error?.details && typeof error.details === 'object' ? error.details : {}),
|
|
97
|
+
pipelineLoadPhase: phase,
|
|
98
|
+
...context,
|
|
99
|
+
};
|
|
100
|
+
return wrapped;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async function withPipelineLoadPhase(phase, context, run) {
|
|
104
|
+
try {
|
|
105
|
+
return await run();
|
|
106
|
+
} catch (error) {
|
|
107
|
+
if (error?.details?.pipelineLoadPhase) {
|
|
108
|
+
throw error;
|
|
109
|
+
}
|
|
110
|
+
throw createPipelineLoadPhaseError(error, phase, context);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
58
114
|
function resolveSingleSpecialTokenId(tokenizer, tokenText, label) {
|
|
59
115
|
const rawTokenIds = tokenizer?.encode?.(tokenText);
|
|
60
116
|
const tokenIds = Array.isArray(rawTokenIds)
|
|
@@ -286,12 +342,17 @@ export class InferencePipeline extends PipelineState {
|
|
|
286
342
|
const fallbackInfo = this.executionV1State.fallbackKernelPath
|
|
287
343
|
? ', fallbackKernelPath=yes'
|
|
288
344
|
: '';
|
|
345
|
+
const laneIntegrity = this.executionV1State.laneIntegrity;
|
|
346
|
+
const laneInfo = laneIntegrity?.status === 'transformed'
|
|
347
|
+
? `, laneIntegrity=transformed(declared=${laneIntegrity.declared.activationDtype}/${laneIntegrity.declared.kvDtype},` +
|
|
348
|
+
`executed=${laneIntegrity.executed.activationDtype}/${laneIntegrity.executed.kvDtype})`
|
|
349
|
+
: '';
|
|
289
350
|
log.info(
|
|
290
351
|
'Pipeline',
|
|
291
352
|
`Execution v1 enabled (steps=${this.executionV1State.resolvedSteps.all.length}, ` +
|
|
292
353
|
`kernelPathInline=${this.executionV1State.runtimeInferencePatch.kernelPath ? 'yes' : 'no'}, ` +
|
|
293
354
|
`pipelineInline=${this.executionV1State.runtimeInferencePatch.pipeline ? 'yes' : 'no'}` +
|
|
294
|
-
`${transformInfo}${fallbackInfo})`
|
|
355
|
+
`${transformInfo}${laneInfo}${fallbackInfo})`
|
|
295
356
|
);
|
|
296
357
|
}
|
|
297
358
|
}
|
|
@@ -377,10 +438,14 @@ export class InferencePipeline extends PipelineState {
|
|
|
377
438
|
const kernelInfo = this.resolvedKernelPath ? `kernelPath=${this.resolvedKernelPath.id}` : 'kernelPath=none';
|
|
378
439
|
log.info('Pipeline', `${cfg.numLayers}L/${cfg.hiddenSize}H/${cfg.numHeads}heads (${cfg.headDim}dim)${moeStr}, ${kernelInfo}`);
|
|
379
440
|
|
|
380
|
-
this.tokenizer = await
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
441
|
+
this.tokenizer = await withPipelineLoadPhase(
|
|
442
|
+
'tokenizer',
|
|
443
|
+
{ modelId: manifest.modelId ?? null },
|
|
444
|
+
() => initTokenizerFromManifest(
|
|
445
|
+
manifest,
|
|
446
|
+
this.baseUrl,
|
|
447
|
+
this.storageContext
|
|
448
|
+
)
|
|
384
449
|
);
|
|
385
450
|
const tokenizerVocabSize = this.tokenizer.getVocabSize();
|
|
386
451
|
if (Number.isFinite(tokenizerVocabSize) && tokenizerVocabSize > 0) {
|
|
@@ -389,23 +454,8 @@ export class InferencePipeline extends PipelineState {
|
|
|
389
454
|
}
|
|
390
455
|
}
|
|
391
456
|
|
|
392
|
-
//
|
|
393
|
-
|
|
394
|
-
const manifestComputeHint = manifest?.quantizationInfo?.compute;
|
|
395
|
-
const resolvedKvDtype = this.runtimeConfig.inference.session?.kvcache?.kvDtype;
|
|
396
|
-
if (
|
|
397
|
-
manifestComputeHint
|
|
398
|
-
&& resolvedKvDtype
|
|
399
|
-
&& String(manifestComputeHint).toLowerCase() !== String(resolvedKvDtype).toLowerCase()
|
|
400
|
-
) {
|
|
401
|
-
log.warn(
|
|
402
|
-
'Pipeline',
|
|
403
|
-
`KV cache kvDtype from execution-v1 resolution (${resolvedKvDtype}) differs from ` +
|
|
404
|
-
`manifest quantizationInfo.compute hint (${manifestComputeHint}). ` +
|
|
405
|
-
`The kernel path dtype contract takes precedence.`
|
|
406
|
-
);
|
|
407
|
-
}
|
|
408
|
-
}
|
|
457
|
+
// Manifest quantizationInfo.compute is the binding lane identity.
|
|
458
|
+
assertManifestComputeLaneBinding({ manifest, runtimeConfig: this.runtimeConfig });
|
|
409
459
|
|
|
410
460
|
// Initialize KV cache
|
|
411
461
|
if (this.modelConfig.decodeStrategy === 'replay_prefill') {
|
|
@@ -464,7 +514,11 @@ export class InferencePipeline extends PipelineState {
|
|
|
464
514
|
}
|
|
465
515
|
|
|
466
516
|
// Load weights
|
|
467
|
-
await
|
|
517
|
+
await withPipelineLoadPhase(
|
|
518
|
+
'loadWeights',
|
|
519
|
+
{ modelId: manifest.modelId ?? null },
|
|
520
|
+
() => this._loadWeights()
|
|
521
|
+
);
|
|
468
522
|
|
|
469
523
|
// Initialize RoPE frequencies
|
|
470
524
|
await this._initRoPE();
|
|
@@ -491,8 +545,10 @@ export class InferencePipeline extends PipelineState {
|
|
|
491
545
|
kernelPathSource: this.kernelPathSource,
|
|
492
546
|
keepF32Weights: this.runtimeConfig.inference.compute.keepF32Weights === true,
|
|
493
547
|
loaderDebug: this.runtimeConfig?.shared?.debug?.loader ?? null,
|
|
494
|
-
perLayerInputSession:
|
|
495
|
-
this.
|
|
548
|
+
perLayerInputSession: resolvePerLayerInputsSession(
|
|
549
|
+
this.modelConfig.perLayerInputsSession ?? null,
|
|
550
|
+
this.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
551
|
+
),
|
|
496
552
|
onProgress: (info) => {
|
|
497
553
|
if (info.stage !== 'layers' && info.stage !== 'shards') {
|
|
498
554
|
log.verbose('Loader', `${info.stage}: ${Math.round(info.progress * 100)}%${info.message ? ` - ${info.message}` : ''}`);
|
|
@@ -917,13 +973,15 @@ export class InferencePipeline extends PipelineState {
|
|
|
917
973
|
* @param {number} [params.softTokenBudget] Per-request soft token budget (Gemma 4 tiers: 70/140/280/560/1120)
|
|
918
974
|
* @returns {Promise<{ text: string, tokens: number[] }>}
|
|
919
975
|
*/
|
|
920
|
-
async transcribeImage({ imageBytes, width, height, prompt, maxTokens, softTokenBudget }) {
|
|
976
|
+
async transcribeImage({ imageBytes, width, height, prompt, maxTokens, softTokenBudget, signal }) {
|
|
977
|
+
assertNotAborted(signal);
|
|
921
978
|
if (!this.visionCapable) {
|
|
922
979
|
throw new Error(
|
|
923
980
|
'Pipeline does not support image transcription (no image_token_id in manifest).'
|
|
924
981
|
);
|
|
925
982
|
}
|
|
926
983
|
await this._ensureVisionWeightsLoaded();
|
|
984
|
+
assertNotAborted(signal);
|
|
927
985
|
|
|
928
986
|
this.reset();
|
|
929
987
|
|
|
@@ -1058,7 +1116,8 @@ export class InferencePipeline extends PipelineState {
|
|
|
1058
1116
|
* @param {number} [params.perFrameSoftTokenBudget] Soft token budget per frame
|
|
1059
1117
|
* @returns {Promise<{ text: string, tokens: number[] }>}
|
|
1060
1118
|
*/
|
|
1061
|
-
async transcribeVideo({ frames, prompt, maxTokens, maxFrames, perFrameSoftTokenBudget }) {
|
|
1119
|
+
async transcribeVideo({ frames, prompt, maxTokens, maxFrames, perFrameSoftTokenBudget, signal }) {
|
|
1120
|
+
assertNotAborted(signal);
|
|
1062
1121
|
if (!this.visionCapable) {
|
|
1063
1122
|
throw new Error(
|
|
1064
1123
|
'Pipeline does not support video transcription (no image_token_id in manifest for vision encoder).'
|
|
@@ -1181,7 +1240,8 @@ export class InferencePipeline extends PipelineState {
|
|
|
1181
1240
|
* @param {number} [params.maxTokens] Max tokens to generate
|
|
1182
1241
|
* @returns {Promise<{ text: string, tokens: number[] }>}
|
|
1183
1242
|
*/
|
|
1184
|
-
async transcribeAudio({ audio, prompt, maxTokens }) {
|
|
1243
|
+
async transcribeAudio({ audio, prompt, maxTokens, signal }) {
|
|
1244
|
+
assertNotAborted(signal);
|
|
1185
1245
|
if (!this.audioCapable) {
|
|
1186
1246
|
throw new Error(
|
|
1187
1247
|
'Pipeline does not support audio transcription (no audio_token_id in manifest).'
|
|
@@ -1350,40 +1410,50 @@ export class InferencePipeline extends PipelineState {
|
|
|
1350
1410
|
|
|
1351
1411
|
|
|
1352
1412
|
generate(prompt, options = {}) {
|
|
1413
|
+
assertNotAborted(options?.signal);
|
|
1353
1414
|
return this.generator.generate(prompt, options);
|
|
1354
1415
|
}
|
|
1355
1416
|
|
|
1356
1417
|
generateTokens(prompt, options = {}) {
|
|
1418
|
+
assertNotAborted(options?.signal);
|
|
1357
1419
|
return this.generator.generateTokens(prompt, options);
|
|
1358
1420
|
}
|
|
1359
1421
|
|
|
1360
1422
|
generateTokenIds(prompt, options = {}) {
|
|
1423
|
+
assertNotAborted(options?.signal);
|
|
1361
1424
|
return this.generator.generateTokenIds(prompt, options);
|
|
1362
1425
|
}
|
|
1363
1426
|
|
|
1364
1427
|
decodeStepLogits(currentIds, options = {}) {
|
|
1428
|
+
assertNotAborted(options?.signal);
|
|
1365
1429
|
return this.generator.decodeStepLogits(currentIds, options);
|
|
1366
1430
|
}
|
|
1367
1431
|
|
|
1368
1432
|
advanceWithToken(tokenId, options = {}) {
|
|
1433
|
+
assertNotAborted(options?.signal);
|
|
1369
1434
|
return this.generator.advanceWithToken(tokenId, options);
|
|
1370
1435
|
}
|
|
1371
1436
|
|
|
1372
1437
|
advanceWithTokenAndEmbedding(tokenId, options = {}) {
|
|
1438
|
+
assertNotAborted(options?.signal);
|
|
1373
1439
|
return this.generator.advanceWithTokenAndEmbedding(tokenId, options);
|
|
1374
1440
|
}
|
|
1375
1441
|
|
|
1376
1442
|
|
|
1377
1443
|
prefillKVOnly(prompt, options = {}) {
|
|
1444
|
+
assertNotAborted(options?.signal);
|
|
1378
1445
|
return this.generator.prefillKVOnly(prompt, options);
|
|
1379
1446
|
}
|
|
1380
1447
|
|
|
1381
1448
|
prefillWithEmbedding(prompt, options = {}) {
|
|
1449
|
+
assertNotAborted(options?.signal);
|
|
1382
1450
|
return this.generator.prefillWithEmbedding(prompt, options);
|
|
1383
1451
|
}
|
|
1384
1452
|
|
|
1385
1453
|
async embed(prompt, options = {}) {
|
|
1454
|
+
assertNotAborted(options?.signal);
|
|
1386
1455
|
const result = await this.prefillWithEmbedding(prompt, options);
|
|
1456
|
+
assertNotAborted(options?.signal);
|
|
1387
1457
|
return {
|
|
1388
1458
|
embedding: result.embedding,
|
|
1389
1459
|
tokens: result.tokens,
|
|
@@ -1396,15 +1466,144 @@ export class InferencePipeline extends PipelineState {
|
|
|
1396
1466
|
if (!Array.isArray(prompts)) {
|
|
1397
1467
|
throw new Error('embedBatch expects an array of prompts');
|
|
1398
1468
|
}
|
|
1469
|
+
assertNotAborted(options?.signal);
|
|
1399
1470
|
const batchOptions = { ...options, __skipStateSnapshot: true };
|
|
1400
1471
|
const outputs = [];
|
|
1401
1472
|
for (const prompt of prompts) {
|
|
1473
|
+
// Check between every prompt so a superseded revision drops the rest.
|
|
1474
|
+
assertNotAborted(options?.signal);
|
|
1402
1475
|
outputs.push(await this.embed(prompt, batchOptions));
|
|
1403
1476
|
this.resetForBatch();
|
|
1404
1477
|
}
|
|
1405
1478
|
return outputs;
|
|
1406
1479
|
}
|
|
1407
1480
|
|
|
1481
|
+
// Run the vision encoder over a single image and return a mean-pooled
|
|
1482
|
+
// embedding in the model's text-hidden-size space. Decoder responsibility
|
|
1483
|
+
// (jpeg/png -> RGBA pixels) belongs to the caller; this method takes
|
|
1484
|
+
// already-decoded pixel data.
|
|
1485
|
+
async embedImage({ pixels, width, height, softTokenBudget, signal } = {}) {
|
|
1486
|
+
assertNotAborted(signal);
|
|
1487
|
+
if (!this.visionCapable) {
|
|
1488
|
+
throw new Error(
|
|
1489
|
+
'Pipeline does not support image embedding (no image_token_id in manifest).'
|
|
1490
|
+
);
|
|
1491
|
+
}
|
|
1492
|
+
if (pixels == null) {
|
|
1493
|
+
throw new Error('[Pipeline] embedImage: pixels are required.');
|
|
1494
|
+
}
|
|
1495
|
+
if (!Number.isFinite(width) || width <= 0 || !Number.isFinite(height) || height <= 0) {
|
|
1496
|
+
throw new Error('[Pipeline] embedImage: width and height must be positive integers.');
|
|
1497
|
+
}
|
|
1498
|
+
await this._ensureVisionWeightsLoaded();
|
|
1499
|
+
this.reset();
|
|
1500
|
+
|
|
1501
|
+
const { encodeImage } = await import('./vision/index.js');
|
|
1502
|
+
const encodeResult = await encodeImage({
|
|
1503
|
+
pixels,
|
|
1504
|
+
width,
|
|
1505
|
+
height,
|
|
1506
|
+
visionConfig: this.visionConfig,
|
|
1507
|
+
weights: this.visionWeights,
|
|
1508
|
+
softTokenBudget,
|
|
1509
|
+
});
|
|
1510
|
+
|
|
1511
|
+
const hiddenSize = this.modelConfig.hiddenSize;
|
|
1512
|
+
const numTokens = encodeResult.numTokens;
|
|
1513
|
+
if (!Number.isFinite(numTokens) || numTokens < 1) {
|
|
1514
|
+
releaseBuffer(encodeResult.features);
|
|
1515
|
+
throw new Error(`[Pipeline] embedImage: encoder produced ${numTokens} soft tokens; expected >= 1.`);
|
|
1516
|
+
}
|
|
1517
|
+
try {
|
|
1518
|
+
const bytes = await readBuffer(
|
|
1519
|
+
encodeResult.features,
|
|
1520
|
+
numTokens * hiddenSize * Float32Array.BYTES_PER_ELEMENT
|
|
1521
|
+
);
|
|
1522
|
+
const features = new Float32Array(bytes);
|
|
1523
|
+
const pooled = new Float32Array(hiddenSize);
|
|
1524
|
+
for (let t = 0; t < numTokens; t++) {
|
|
1525
|
+
const base = t * hiddenSize;
|
|
1526
|
+
for (let d = 0; d < hiddenSize; d++) {
|
|
1527
|
+
pooled[d] += features[base + d];
|
|
1528
|
+
}
|
|
1529
|
+
}
|
|
1530
|
+
const inv = 1 / numTokens;
|
|
1531
|
+
for (let d = 0; d < hiddenSize; d++) pooled[d] *= inv;
|
|
1532
|
+
return {
|
|
1533
|
+
embedding: pooled,
|
|
1534
|
+
embeddingDim: hiddenSize,
|
|
1535
|
+
numTokens,
|
|
1536
|
+
embeddingMode: 'mean',
|
|
1537
|
+
};
|
|
1538
|
+
} finally {
|
|
1539
|
+
releaseBuffer(encodeResult.features);
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
|
|
1543
|
+
// Run the audio encoder over a single PCM segment and return a mean-pooled
|
|
1544
|
+
// embedding in the model's audio-projection-output space (which equals the
|
|
1545
|
+
// text hidden size in Gemma 4). Decoder responsibility (webm/opus/wav ->
|
|
1546
|
+
// Float32 PCM at the model's expected sample rate) belongs to the caller.
|
|
1547
|
+
async embedAudio({ audio, signal } = {}) {
|
|
1548
|
+
assertNotAborted(signal);
|
|
1549
|
+
if (!this.audioCapable) {
|
|
1550
|
+
throw new Error(
|
|
1551
|
+
'Pipeline does not support audio embedding (no audio_token_id in manifest).'
|
|
1552
|
+
);
|
|
1553
|
+
}
|
|
1554
|
+
if (audio == null) {
|
|
1555
|
+
throw new Error('[Pipeline] embedAudio: audio is required.');
|
|
1556
|
+
}
|
|
1557
|
+
await this._ensureAudioWeightsLoaded();
|
|
1558
|
+
this.reset();
|
|
1559
|
+
|
|
1560
|
+
const { extractLogMelSpectrogram } = await import('./audio/mel.js');
|
|
1561
|
+
const { encodeAudio } = await import('./audio/index.js');
|
|
1562
|
+
const { features: melFeatures, numFrames, nMels } = extractLogMelSpectrogram(audio);
|
|
1563
|
+
const encodeResult = await encodeAudio({
|
|
1564
|
+
melFeatures,
|
|
1565
|
+
numFrames,
|
|
1566
|
+
nMels,
|
|
1567
|
+
audioConfig: this.audioConfig,
|
|
1568
|
+
weights: this.audioWeights,
|
|
1569
|
+
});
|
|
1570
|
+
|
|
1571
|
+
const hiddenSize = Number(this.audioConfig?.outputProjDims ?? this.modelConfig?.hiddenSize);
|
|
1572
|
+
const numTokens = encodeResult.numTokens;
|
|
1573
|
+
if (!Number.isFinite(hiddenSize) || hiddenSize < 1) {
|
|
1574
|
+
releaseBuffer(encodeResult.features);
|
|
1575
|
+
throw new Error('[Pipeline] embedAudio: audioConfig.outputProjDims is missing or invalid.');
|
|
1576
|
+
}
|
|
1577
|
+
if (!Number.isFinite(numTokens) || numTokens < 1) {
|
|
1578
|
+
releaseBuffer(encodeResult.features);
|
|
1579
|
+
throw new Error(`[Pipeline] embedAudio: encoder produced ${numTokens} tokens; expected >= 1.`);
|
|
1580
|
+
}
|
|
1581
|
+
try {
|
|
1582
|
+
const bytes = await readBuffer(
|
|
1583
|
+
encodeResult.features,
|
|
1584
|
+
numTokens * hiddenSize * Float32Array.BYTES_PER_ELEMENT
|
|
1585
|
+
);
|
|
1586
|
+
const features = new Float32Array(bytes);
|
|
1587
|
+
const pooled = new Float32Array(hiddenSize);
|
|
1588
|
+
for (let t = 0; t < numTokens; t++) {
|
|
1589
|
+
const base = t * hiddenSize;
|
|
1590
|
+
for (let d = 0; d < hiddenSize; d++) {
|
|
1591
|
+
pooled[d] += features[base + d];
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
const inv = 1 / numTokens;
|
|
1595
|
+
for (let d = 0; d < hiddenSize; d++) pooled[d] *= inv;
|
|
1596
|
+
return {
|
|
1597
|
+
embedding: pooled,
|
|
1598
|
+
embeddingDim: hiddenSize,
|
|
1599
|
+
numTokens,
|
|
1600
|
+
embeddingMode: 'mean',
|
|
1601
|
+
};
|
|
1602
|
+
} finally {
|
|
1603
|
+
releaseBuffer(encodeResult.features);
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
|
|
1408
1607
|
prefillWithLogits(prompt, options = {}) {
|
|
1409
1608
|
return this.generator.prefillWithLogits(prompt, options);
|
|
1410
1609
|
}
|
|
@@ -1548,6 +1747,8 @@ export class InferencePipeline extends PipelineState {
|
|
|
1548
1747
|
|
|
1549
1748
|
|
|
1550
1749
|
async unload() {
|
|
1750
|
+
const storageContext = this.storageContext;
|
|
1751
|
+
this.storageContext = null;
|
|
1551
1752
|
await destroyEmulation(this.emulation);
|
|
1552
1753
|
this.emulation = null;
|
|
1553
1754
|
this.decodeRing?.release();
|
|
@@ -1570,6 +1771,9 @@ export class InferencePipeline extends PipelineState {
|
|
|
1570
1771
|
this.sampleReadbackBuffer.destroy();
|
|
1571
1772
|
this.sampleReadbackBuffer = null;
|
|
1572
1773
|
}
|
|
1774
|
+
if (typeof storageContext?.close === 'function') {
|
|
1775
|
+
await storageContext.close();
|
|
1776
|
+
}
|
|
1573
1777
|
this.isLoaded = false;
|
|
1574
1778
|
this.currentSeqLen = 0;
|
|
1575
1779
|
restorePipelineContexts(this);
|
|
@@ -81,113 +81,6 @@ export async function encodeImage(params) {
|
|
|
81
81
|
};
|
|
82
82
|
}
|
|
83
83
|
|
|
84
|
-
/**
|
|
85
|
-
* Inject visual tokens into text token embeddings.
|
|
86
|
-
*
|
|
87
|
-
* Replaces positions in the embedding sequence where image_token_id appears
|
|
88
|
-
* with the encoded visual features from the vision encoder.
|
|
89
|
-
*
|
|
90
|
-
* For Qwen3-VL with DeepStack, visual tokens are injected at specific decoder
|
|
91
|
-
* layers (deepstackVisualIndexes), not at the input embedding level.
|
|
92
|
-
* This function handles the simpler input-level injection case.
|
|
93
|
-
* DeepStack injection is handled in the decoder layer loop.
|
|
94
|
-
*
|
|
95
|
-
* @param {object} params
|
|
96
|
-
* @param {Float32Array} params.textEmbeddings [seqLen, hiddenSize]
|
|
97
|
-
* @param {Int32Array} params.tokenIds [seqLen]
|
|
98
|
-
* @param {GPUBuffer} params.visualFeatures [numVisualTokens, outHiddenSize]
|
|
99
|
-
* @param {number} params.numVisualTokens Number of visual tokens
|
|
100
|
-
* @param {number} params.imageTokenId Token ID marking image positions
|
|
101
|
-
* @param {number} params.hiddenSize Text model hidden size
|
|
102
|
-
* @returns {{ mergedEmbeddings: Float32Array, mergedLength: number }}
|
|
103
|
-
*/
|
|
104
|
-
export function mergeVisualTokens(params) {
|
|
105
|
-
const {
|
|
106
|
-
textEmbeddings, tokenIds, visualFeatures,
|
|
107
|
-
numVisualTokens, imageTokenId, hiddenSize,
|
|
108
|
-
} = params;
|
|
109
|
-
|
|
110
|
-
// Count image token positions.
|
|
111
|
-
const imagePositions = [];
|
|
112
|
-
for (let i = 0; i < tokenIds.length; i++) {
|
|
113
|
-
if (tokenIds[i] === imageTokenId) {
|
|
114
|
-
imagePositions.push(i);
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
if (imagePositions.length === 0) {
|
|
119
|
-
log.debug('Vision', 'mergeVisualTokens: no image tokens found, returning text-only');
|
|
120
|
-
return { mergedEmbeddings: textEmbeddings, mergedLength: tokenIds.length };
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
log.debug('Vision', `mergeVisualTokens: replacing ${imagePositions.length} image tokens with ${numVisualTokens} visual tokens`);
|
|
124
|
-
|
|
125
|
-
// The merged sequence replaces contiguous image_token_id runs with visual features.
|
|
126
|
-
// For Qwen3-VL: image tokens appear as a block between vision_start and vision_end tokens.
|
|
127
|
-
// The visual features replace the entire image token block.
|
|
128
|
-
|
|
129
|
-
// Find contiguous image token ranges.
|
|
130
|
-
const ranges = [];
|
|
131
|
-
let rangeStart = imagePositions[0];
|
|
132
|
-
let rangeEnd = imagePositions[0];
|
|
133
|
-
for (let i = 1; i < imagePositions.length; i++) {
|
|
134
|
-
if (imagePositions[i] === rangeEnd + 1) {
|
|
135
|
-
rangeEnd = imagePositions[i];
|
|
136
|
-
} else {
|
|
137
|
-
ranges.push([rangeStart, rangeEnd]);
|
|
138
|
-
rangeStart = imagePositions[i];
|
|
139
|
-
rangeEnd = imagePositions[i];
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
ranges.push([rangeStart, rangeEnd]);
|
|
143
|
-
|
|
144
|
-
// Build merged sequence: text tokens (non-image) + visual tokens replacing each range.
|
|
145
|
-
const textLen = tokenIds.length;
|
|
146
|
-
const replacedCount = imagePositions.length;
|
|
147
|
-
const mergedLength = textLen - replacedCount + numVisualTokens;
|
|
148
|
-
const merged = new Float32Array(mergedLength * hiddenSize);
|
|
149
|
-
|
|
150
|
-
let srcPos = 0;
|
|
151
|
-
let dstPos = 0;
|
|
152
|
-
let visualOffset = 0;
|
|
153
|
-
|
|
154
|
-
for (const [start, end] of ranges) {
|
|
155
|
-
// Copy text tokens before this range.
|
|
156
|
-
const textBefore = start - srcPos;
|
|
157
|
-
if (textBefore > 0) {
|
|
158
|
-
merged.set(
|
|
159
|
-
textEmbeddings.subarray(srcPos * hiddenSize, start * hiddenSize),
|
|
160
|
-
dstPos * hiddenSize,
|
|
161
|
-
);
|
|
162
|
-
dstPos += textBefore;
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
// Insert visual tokens replacing this range.
|
|
166
|
-
const rangeLen = end - start + 1;
|
|
167
|
-
const tokensToInsert = Math.min(numVisualTokens - visualOffset, rangeLen);
|
|
168
|
-
// Copy from visual features buffer (CPU side).
|
|
169
|
-
for (let i = 0; i < tokensToInsert; i++) {
|
|
170
|
-
for (let d = 0; d < hiddenSize; d++) {
|
|
171
|
-
merged[(dstPos + i) * hiddenSize + d] = visualFeatures[(visualOffset + i) * hiddenSize + d];
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
dstPos += tokensToInsert;
|
|
175
|
-
visualOffset += tokensToInsert;
|
|
176
|
-
srcPos = end + 1;
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
// Copy remaining text tokens after last range.
|
|
180
|
-
if (srcPos < textLen) {
|
|
181
|
-
merged.set(
|
|
182
|
-
textEmbeddings.subarray(srcPos * hiddenSize, textLen * hiddenSize),
|
|
183
|
-
dstPos * hiddenSize,
|
|
184
|
-
);
|
|
185
|
-
dstPos += textLen - srcPos;
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
return { mergedEmbeddings: merged, mergedLength: dstPos };
|
|
189
|
-
}
|
|
190
|
-
|
|
191
84
|
/**
|
|
192
85
|
* @typedef {object} VisionEncodeResult
|
|
193
86
|
* @property {GPUBuffer} features Encoded visual tokens [numTokens, outHiddenSize]
|
|
@@ -80,10 +80,3 @@ export interface CreateRuntimeModelContractOptions
|
|
|
80
80
|
export declare function createRuntimeModelContract(
|
|
81
81
|
options: CreateRuntimeModelContractOptions
|
|
82
82
|
): RuntimeModelContract;
|
|
83
|
-
|
|
84
|
-
export declare function createRuntimeModelFromManifest<T extends Record<string, unknown>>(
|
|
85
|
-
manifest: T,
|
|
86
|
-
options?: { sourceFormat?: string | null }
|
|
87
|
-
): T & RuntimeModelContract;
|
|
88
|
-
|
|
89
|
-
export declare function isRuntimeModelContract(value: unknown): value is RuntimeModelContract;
|
|
@@ -127,26 +127,3 @@ export function createRuntimeModelContract(options = {}) {
|
|
|
127
127
|
};
|
|
128
128
|
}
|
|
129
129
|
|
|
130
|
-
export function createRuntimeModelFromManifest(manifest, options = {}) {
|
|
131
|
-
if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) {
|
|
132
|
-
throw new Error('createRuntimeModelFromManifest requires a manifest-like object.');
|
|
133
|
-
}
|
|
134
|
-
return {
|
|
135
|
-
...manifest,
|
|
136
|
-
kind: 'runtime-model',
|
|
137
|
-
sourceFormat: normalizeText(options.sourceFormat).toLowerCase()
|
|
138
|
-
|| normalizeText(manifest?.sourceFormat).toLowerCase()
|
|
139
|
-
|| 'rdrr',
|
|
140
|
-
};
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
export function isRuntimeModelContract(value) {
|
|
144
|
-
return value != null
|
|
145
|
-
&& typeof value === 'object'
|
|
146
|
-
&& !Array.isArray(value)
|
|
147
|
-
&& Array.isArray(value.shards)
|
|
148
|
-
&& typeof value.modelId === 'string'
|
|
149
|
-
&& value.modelId.trim().length > 0
|
|
150
|
-
&& value.inference != null
|
|
151
|
-
&& typeof value.inference === 'object';
|
|
152
|
-
}
|
|
@@ -53,6 +53,8 @@ export interface InferenceHarnessOptions {
|
|
|
53
53
|
baseUrl?: string;
|
|
54
54
|
/** Runtime overrides for kernel selection */
|
|
55
55
|
runtime?: RuntimeOverrides;
|
|
56
|
+
/** Explicit source loading mode from the command/suite request */
|
|
57
|
+
loadMode?: 'opfs' | 'http' | 'memory' | 'file' | null;
|
|
56
58
|
/** Progress callback */
|
|
57
59
|
onProgress?: (phase: string, progress: number, detail?: string) => void;
|
|
58
60
|
/** Log function (default: debug log) */
|
|
@@ -112,37 +114,19 @@ export declare function parseRuntimeOverridesFromURL(
|
|
|
112
114
|
* @param log - Optional logging function
|
|
113
115
|
* @returns Async function that loads a shard by index
|
|
114
116
|
*/
|
|
115
|
-
export declare function createHttpShardLoader(
|
|
116
|
-
baseUrl: string,
|
|
117
|
-
manifest: RDRRManifest,
|
|
118
|
-
log?: (msg: string, level?: string) => void
|
|
119
|
-
): (idx: number) => Promise<Uint8Array>;
|
|
120
|
-
|
|
121
117
|
export declare function createHarnessShardStorageContext(
|
|
122
118
|
modelUrl: string,
|
|
123
119
|
manifest: RDRRManifest,
|
|
124
|
-
log?: (msg: string, level?: string) => void
|
|
120
|
+
log?: (msg: string, level?: string) => void,
|
|
121
|
+
options?: {
|
|
122
|
+
loadMode?: 'opfs' | 'http' | 'memory' | 'file' | null;
|
|
123
|
+
}
|
|
125
124
|
): PipelineStorageContext;
|
|
126
125
|
|
|
127
126
|
// ============================================================================
|
|
128
127
|
// Pipeline Initialization
|
|
129
128
|
// ============================================================================
|
|
130
129
|
|
|
131
|
-
/**
|
|
132
|
-
* Fetch and parse a model manifest from a URL.
|
|
133
|
-
*
|
|
134
|
-
* @param manifestUrl - URL to manifest.json
|
|
135
|
-
* @returns Parsed manifest
|
|
136
|
-
*/
|
|
137
|
-
export declare function fetchManifest(manifestUrl: string): Promise<RDRRManifest>;
|
|
138
|
-
|
|
139
|
-
/**
|
|
140
|
-
* Initialize the WebGPU device and return capabilities.
|
|
141
|
-
*
|
|
142
|
-
* @returns Kernel capabilities
|
|
143
|
-
*/
|
|
144
|
-
export declare function initializeDevice(): Promise<KernelCapabilities>;
|
|
145
|
-
|
|
146
130
|
/**
|
|
147
131
|
* Initialize a complete inference pipeline from a model URL.
|
|
148
132
|
*
|