doppler-gpu 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +19 -15
- package/package.json +77 -29
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +64 -6
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +524 -350
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +12 -2
- package/src/config/schema/doppler.schema.js +108 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +68 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +76 -0
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +107 -0
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +3 -21
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +6 -5
- package/src/gpu/kernels/index.js +3 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +12 -0
- package/src/inference/browser-harness-text-helpers.js +223 -14
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +264 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +22 -27
- package/src/inference/pipelines/text/embed.js +0 -39
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +4 -5
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +8 -3
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +223 -547
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +5 -2
- package/src/inference/pipelines/text/init.js +183 -724
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.js +2 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +120 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +223 -27
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +13 -3
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +11 -3
- package/src/storage/artifact-storage-context.js +231 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +21 -24
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +2 -0
- package/src/tooling/source-runtime-bundle.js +3 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
getActiveKernelPathPolicy,
|
|
12
12
|
} from '../config/kernel-path-loader.js';
|
|
13
13
|
import { validateTrainingMetricsReport } from '../config/schema/training-metrics.schema.js';
|
|
14
|
+
import { modelSupportsEmbedding } from '../config/schema/manifest.schema.js';
|
|
14
15
|
import {
|
|
15
16
|
resolveReportTimestamp,
|
|
16
17
|
resolveRuntime,
|
|
@@ -33,6 +34,7 @@ import {
|
|
|
33
34
|
normalizeWorkloadType,
|
|
34
35
|
assertDiffusionPerformanceArtifact,
|
|
35
36
|
toTimingNumber,
|
|
37
|
+
buildFirstLoadComposition,
|
|
36
38
|
safeToFixed,
|
|
37
39
|
sampleTimingNumber,
|
|
38
40
|
buildCanonicalTiming,
|
|
@@ -56,6 +58,8 @@ import {
|
|
|
56
58
|
runEnergySuite,
|
|
57
59
|
} from './browser-harness-diffusion-energy-suites.js';
|
|
58
60
|
import { collectTrainingArtifactsFromSuiteResult } from './browser-harness-report-helpers.js';
|
|
61
|
+
import { sha256Hex } from '../utils/sha256.js';
|
|
62
|
+
import { stableSortObject } from '../utils/stable-sort-object.js';
|
|
59
63
|
|
|
60
64
|
const TRAINING_SUITE_MODULE_PATH = '../experimental/training/suite.js';
|
|
61
65
|
let trainingSuiteModulePromise = null;
|
|
@@ -67,15 +71,6 @@ async function loadTrainingSuiteModule() {
|
|
|
67
71
|
return trainingSuiteModulePromise;
|
|
68
72
|
}
|
|
69
73
|
|
|
70
|
-
/**
|
|
71
|
-
* Clear the cached training suite module promise.
|
|
72
|
-
* Useful for testing or when the training module needs to be re-imported
|
|
73
|
-
* after configuration changes.
|
|
74
|
-
*/
|
|
75
|
-
export function clearTrainingSuiteModule() {
|
|
76
|
-
trainingSuiteModulePromise = null;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
74
|
export async function runTrainingSuite(options = {}) {
|
|
80
75
|
const module = await loadTrainingSuiteModule();
|
|
81
76
|
return module.runTrainingSuite(options);
|
|
@@ -95,48 +90,6 @@ async function runTrainingBenchSuite(options = {}) {
|
|
|
95
90
|
return module.runTrainingBenchSuite(options);
|
|
96
91
|
}
|
|
97
92
|
|
|
98
|
-
export async function initializeBrowserHarness(options = {}) {
|
|
99
|
-
const { modelUrl, onProgress, log } = options;
|
|
100
|
-
if (!modelUrl) {
|
|
101
|
-
throw new Error('modelUrl is required');
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
const runtime = resolveRuntime(options);
|
|
105
|
-
const result = await initializeInference(modelUrl, {
|
|
106
|
-
runtime,
|
|
107
|
-
onProgress,
|
|
108
|
-
log,
|
|
109
|
-
});
|
|
110
|
-
|
|
111
|
-
return { ...result, runtime };
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
export async function saveBrowserReport(modelId, report, options = {}) {
|
|
115
|
-
return saveReport(modelId, report, options);
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
export async function runBrowserHarness(options = {}) {
|
|
119
|
-
const harness = await initializeBrowserHarness(options);
|
|
120
|
-
const reportTimestamp = resolveReportTimestamp(options.timestamp, 'runBrowserHarness timestamp');
|
|
121
|
-
const modelId = options.modelId || harness.manifest?.modelId || 'unknown';
|
|
122
|
-
|
|
123
|
-
let report = options.report || null;
|
|
124
|
-
if (!report && typeof options.buildReport === 'function') {
|
|
125
|
-
report = await options.buildReport(harness);
|
|
126
|
-
}
|
|
127
|
-
if (!report) {
|
|
128
|
-
report = {
|
|
129
|
-
modelId,
|
|
130
|
-
timestamp: reportTimestamp,
|
|
131
|
-
};
|
|
132
|
-
} else if (!report.timestamp) {
|
|
133
|
-
report = { ...report, timestamp: reportTimestamp };
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
const reportInfo = await saveReport(modelId, report, { timestamp: report.timestamp || reportTimestamp });
|
|
137
|
-
return { ...harness, report, reportInfo };
|
|
138
|
-
}
|
|
139
|
-
|
|
140
93
|
const BROWSER_WORKLOAD_SET = Object.freeze([
|
|
141
94
|
'kernels',
|
|
142
95
|
'inference',
|
|
@@ -296,6 +249,147 @@ function resolveDispatchSuite(mode, workload) {
|
|
|
296
249
|
return workload;
|
|
297
250
|
}
|
|
298
251
|
|
|
252
|
+
function stableJson(value) {
|
|
253
|
+
return JSON.stringify(stableSortObject(value)) ?? 'null';
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function hashStableJson(value) {
|
|
257
|
+
return `sha256:${sha256Hex(stableJson(value))}`;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
export function resolveExecutionGraphHash(manifest) {
|
|
261
|
+
const execution = manifest?.inference?.execution;
|
|
262
|
+
if (!execution || typeof execution !== 'object') {
|
|
263
|
+
return null;
|
|
264
|
+
}
|
|
265
|
+
return hashStableJson(execution);
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function buildPerStepTokenProof(tokenIds) {
|
|
269
|
+
return tokenIds.map((tokenId, index) => ({
|
|
270
|
+
index,
|
|
271
|
+
tokenId,
|
|
272
|
+
tokenHash: hashStableJson({ index, tokenId }),
|
|
273
|
+
}));
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function buildKvCacheTranscriptSeed(kvCache, byteProof = null) {
|
|
277
|
+
const source = kvCache && typeof kvCache === 'object' ? kvCache : null;
|
|
278
|
+
const proof = byteProof && typeof byteProof === 'object' ? byteProof : null;
|
|
279
|
+
const seed = {
|
|
280
|
+
mode: proof ? 'stats+sha256-layer-kv-bytes' : (source ? 'stats' : 'not-captured'),
|
|
281
|
+
layout: typeof source?.layout === 'string' ? source.layout : null,
|
|
282
|
+
kvDtype: typeof source?.kvDtype === 'string' ? source.kvDtype : null,
|
|
283
|
+
seqLen: Number.isFinite(source?.seqLen) ? source.seqLen : null,
|
|
284
|
+
maxSeqLen: Number.isFinite(source?.maxSeqLen) ? source.maxSeqLen : null,
|
|
285
|
+
usedBytes: Number.isFinite(source?.usedBytes) ? source.usedBytes : null,
|
|
286
|
+
allocatedBytes: Number.isFinite(source?.allocatedBytes) ? source.allocatedBytes : null,
|
|
287
|
+
counters: source?.counters ?? null,
|
|
288
|
+
byteDigestMode: typeof proof?.mode === 'string' ? proof.mode : null,
|
|
289
|
+
byteDigest: typeof proof?.digest === 'string' ? proof.digest : null,
|
|
290
|
+
byteDigests: Array.isArray(proof?.layers) ? proof.layers : null,
|
|
291
|
+
};
|
|
292
|
+
return {
|
|
293
|
+
...seed,
|
|
294
|
+
stateHash: hashStableJson(seed),
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
export function buildReferenceTranscriptSeed(run, context = {}) {
|
|
299
|
+
const promptPayload = run.promptInput ?? run.prompt ?? null;
|
|
300
|
+
const outputText = typeof run.output === 'string' ? run.output : '';
|
|
301
|
+
const tokenIds = Array.isArray(run.tokenIds)
|
|
302
|
+
? run.tokenIds.map((value) => Number(value)).filter((value) => Number.isInteger(value))
|
|
303
|
+
: [];
|
|
304
|
+
const promptTokenIds = Array.isArray(run.promptTokenIds)
|
|
305
|
+
? run.promptTokenIds.map((value) => Number(value)).filter((value) => Number.isInteger(value))
|
|
306
|
+
: null;
|
|
307
|
+
const logitsDigests = Array.isArray(run.logitsDigests)
|
|
308
|
+
? run.logitsDigests
|
|
309
|
+
: [];
|
|
310
|
+
const hasCompleteLogitsDigests = logitsDigests.length === tokenIds.length && tokenIds.length > 0;
|
|
311
|
+
const transcript = {
|
|
312
|
+
schema: 'doppler.reference-transcript/v1',
|
|
313
|
+
source: {
|
|
314
|
+
kind: 'inline-browser-suite',
|
|
315
|
+
path: 'inline',
|
|
316
|
+
hash: 'sha256:' + '0'.repeat(64),
|
|
317
|
+
},
|
|
318
|
+
executionGraphHash: context.executionGraphHash ?? null,
|
|
319
|
+
surface: 'browser-webgpu',
|
|
320
|
+
prompt: {
|
|
321
|
+
identity: typeof run.prompt === 'string' && run.prompt.trim() ? run.prompt : 'promptInput',
|
|
322
|
+
hash: hashStableJson(promptPayload),
|
|
323
|
+
tokenIdsHash: promptTokenIds ? hashStableJson(promptTokenIds) : null,
|
|
324
|
+
tokenCount: promptTokenIds ? promptTokenIds.length : null,
|
|
325
|
+
},
|
|
326
|
+
output: {
|
|
327
|
+
textHash: `sha256:${sha256Hex(outputText)}`,
|
|
328
|
+
tokensGenerated: tokenIds.length,
|
|
329
|
+
stopReason: typeof run.phase?.stopReason === 'string' ? run.phase.stopReason : 'unknown',
|
|
330
|
+
stopTokenId: Number.isInteger(run.phase?.stopTokenId) ? run.phase.stopTokenId : null,
|
|
331
|
+
},
|
|
332
|
+
tokens: {
|
|
333
|
+
ids: tokenIds,
|
|
334
|
+
generatedTokenIdsHash: hashStableJson(tokenIds),
|
|
335
|
+
generatedTextHash: `sha256:${sha256Hex(outputText)}`,
|
|
336
|
+
preview: Array.isArray(run.tokenDiagnostics?.preview) ? run.tokenDiagnostics.preview : [],
|
|
337
|
+
perStep: buildPerStepTokenProof(tokenIds),
|
|
338
|
+
coverage: {
|
|
339
|
+
mode: 'full-token-ids',
|
|
340
|
+
omitted: 0,
|
|
341
|
+
},
|
|
342
|
+
},
|
|
343
|
+
phase: {
|
|
344
|
+
prefillMs: Number.isFinite(run.phase?.prefillMs) ? run.phase.prefillMs : null,
|
|
345
|
+
decodeMs: Number.isFinite(run.phase?.decodeMs) ? run.phase.decodeMs : null,
|
|
346
|
+
prefillTokens: Number.isFinite(run.phase?.prefillTokens) ? run.phase.prefillTokens : null,
|
|
347
|
+
decodeTokens: Number.isFinite(run.phase?.decodeTokens) ? run.phase.decodeTokens : null,
|
|
348
|
+
},
|
|
349
|
+
kvCache: buildKvCacheTranscriptSeed(
|
|
350
|
+
run.phase?.kvCache ?? context.kvCache ?? null,
|
|
351
|
+
run.kvCacheByteProof ?? null
|
|
352
|
+
),
|
|
353
|
+
logits: hasCompleteLogitsDigests ? {
|
|
354
|
+
mode: 'sha256-per-step',
|
|
355
|
+
perStepDigests: logitsDigests.map((entry) => entry.digest),
|
|
356
|
+
steps: logitsDigests,
|
|
357
|
+
} : {
|
|
358
|
+
mode: 'not-captured',
|
|
359
|
+
reason: logitsDigests.length > 0
|
|
360
|
+
? 'Per-step logits digest count did not match generated token count.'
|
|
361
|
+
: 'Per-step logits digests were not requested for this browser harness run.',
|
|
362
|
+
perStepDigests: null,
|
|
363
|
+
},
|
|
364
|
+
tolerance: {
|
|
365
|
+
tokenPolicy: 'exact generated token IDs',
|
|
366
|
+
logitsPolicy: hasCompleteLogitsDigests
|
|
367
|
+
? 'exact sha256 digest per generated step over finalized f32 logits before sampling'
|
|
368
|
+
: 'not captured',
|
|
369
|
+
kvPolicy: run.kvCacheByteProof
|
|
370
|
+
? 'exact sha256 digest over used KV cache bytes by layer/key/value'
|
|
371
|
+
: 'metadata hash only; KV tensor bytes are not read back by default',
|
|
372
|
+
},
|
|
373
|
+
};
|
|
374
|
+
return {
|
|
375
|
+
...transcript,
|
|
376
|
+
source: {
|
|
377
|
+
...transcript.source,
|
|
378
|
+
hash: hashStableJson({
|
|
379
|
+
prompt: transcript.prompt,
|
|
380
|
+
output: transcript.output,
|
|
381
|
+
tokens: {
|
|
382
|
+
generatedTokenIdsHash: transcript.tokens.generatedTokenIdsHash,
|
|
383
|
+
generatedTextHash: transcript.tokens.generatedTextHash,
|
|
384
|
+
},
|
|
385
|
+
phase: transcript.phase,
|
|
386
|
+
kvCache: transcript.kvCache,
|
|
387
|
+
logits: transcript.logits,
|
|
388
|
+
}),
|
|
389
|
+
},
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
|
|
299
393
|
async function runKernelSuite(options = {}) {
|
|
300
394
|
const startTime = performance.now();
|
|
301
395
|
const { testHarness, initGPU } = await import('../../tests/kernels/browser/test-page.js');
|
|
@@ -324,23 +418,33 @@ async function runKernelSuite(options = {}) {
|
|
|
324
418
|
|
|
325
419
|
async function runInferenceSuite(options = {}) {
|
|
326
420
|
const startTime = performance.now();
|
|
327
|
-
const
|
|
421
|
+
const cacheMode = normalizeCacheMode(options.cacheMode);
|
|
422
|
+
const loadMode = normalizeLoadMode(options.loadMode, !!options.modelUrl, options.modelUrl);
|
|
423
|
+
const harness = await withHarnessPhase(
|
|
424
|
+
'inference.initializeSuiteModel',
|
|
425
|
+
{
|
|
426
|
+
modelId: options.modelId ?? null,
|
|
427
|
+
loadMode,
|
|
428
|
+
cacheMode,
|
|
429
|
+
},
|
|
430
|
+
() => initializeSuiteModel(options)
|
|
431
|
+
);
|
|
328
432
|
const runtimeConfig = getRuntimeConfig();
|
|
329
433
|
const modelType = harness.manifest?.modelType || 'transformer';
|
|
330
|
-
|
|
434
|
+
const supportsEmbedding = modelSupportsEmbedding(harness.manifest);
|
|
435
|
+
if (options.expectedModelType === 'embedding' && !supportsEmbedding) {
|
|
331
436
|
throw new Error(
|
|
332
|
-
`Expected an embedding model for workload "${options.workload || 'inference'}", got "${modelType}"
|
|
437
|
+
`Expected an embedding-capable model for workload "${options.workload || 'inference'}", got modelType="${modelType}". ` +
|
|
438
|
+
`Set inference.supportsEmbedding=true in the manifest for text-generation models that should expose pipeline.embed().`
|
|
333
439
|
);
|
|
334
440
|
}
|
|
335
|
-
const cacheMode = normalizeCacheMode(options.cacheMode);
|
|
336
|
-
const loadMode = normalizeLoadMode(options.loadMode, !!options.modelUrl, options.modelUrl);
|
|
337
441
|
const safeModelLoadMs = toTimingNumber(harness.modelLoadMs, 0);
|
|
338
442
|
|
|
339
443
|
let results;
|
|
340
444
|
let output = null;
|
|
341
445
|
let metrics;
|
|
342
446
|
|
|
343
|
-
if (modelType === 'embedding') {
|
|
447
|
+
if (modelType === 'embedding' || (options.workload === 'embedding' && supportsEmbedding)) {
|
|
344
448
|
const run = await runEmbedding(harness.pipeline, runtimeConfig);
|
|
345
449
|
const semantic = await runEmbeddingSemanticChecks(harness.pipeline, options);
|
|
346
450
|
const isValidEmbedding = run.embeddingDim > 0 && run.nonFiniteCount === 0;
|
|
@@ -464,6 +568,8 @@ async function runInferenceSuite(options = {}) {
|
|
|
464
568
|
decodeMs: safeToFixed(run.phase.decodeMs),
|
|
465
569
|
prefillTokens: Math.round(run.phase.prefillTokens),
|
|
466
570
|
decodeTokens: Math.round(run.phase.decodeTokens),
|
|
571
|
+
stopReason: run.phase.stopReason ?? null,
|
|
572
|
+
stopTokenId: Number.isInteger(run.phase.stopTokenId) ? run.phase.stopTokenId : null,
|
|
467
573
|
prefillTokensPerSec: safeToFixed(run.phase.prefillTokensPerSec),
|
|
468
574
|
prefillTokensPerSecTtft: safeToFixed(run.phase.prefillTokensPerSecTtft),
|
|
469
575
|
decodeTokensPerSec: safeToFixed(run.phase.decodeTokensPerSec),
|
|
@@ -476,6 +582,11 @@ async function runInferenceSuite(options = {}) {
|
|
|
476
582
|
kernelPathId: run.phase.kernelPathId,
|
|
477
583
|
kernelPathSource: run.phase.kernelPathSource,
|
|
478
584
|
generationDiagnostics: run.tokenDiagnostics,
|
|
585
|
+
kvCache: run.phase.kvCache ?? null,
|
|
586
|
+
referenceTranscript: buildReferenceTranscriptSeed(run, {
|
|
587
|
+
executionGraphHash: resolveExecutionGraphHash(harness.manifest),
|
|
588
|
+
kvCache: run.phase.kvCache ?? null,
|
|
589
|
+
}),
|
|
479
590
|
operatorDiagnostics: run.phase.operatorDiagnostics ?? null,
|
|
480
591
|
};
|
|
481
592
|
}
|
|
@@ -509,6 +620,11 @@ async function runInferenceSuite(options = {}) {
|
|
|
509
620
|
source: 'doppler',
|
|
510
621
|
prefillSemantics: 'internal_prefill_phase',
|
|
511
622
|
});
|
|
623
|
+
const firstLoad = buildFirstLoadComposition({
|
|
624
|
+
modelLoadMs: timing.modelLoadMs,
|
|
625
|
+
firstTokenMs: timing.firstTokenMs,
|
|
626
|
+
firstResponseMs: timing.firstResponseMs,
|
|
627
|
+
});
|
|
512
628
|
const metricsWithContracts = buildSuiteContractMetrics(
|
|
513
629
|
options.suiteName || 'inference',
|
|
514
630
|
metrics,
|
|
@@ -530,6 +646,7 @@ async function runInferenceSuite(options = {}) {
|
|
|
530
646
|
},
|
|
531
647
|
timing,
|
|
532
648
|
timingDiagnostics,
|
|
649
|
+
firstLoad,
|
|
533
650
|
output,
|
|
534
651
|
metrics: metricsWithContracts,
|
|
535
652
|
memoryStats,
|
|
@@ -584,6 +701,11 @@ async function runBenchSuite(options = {}) {
|
|
|
584
701
|
source: 'doppler',
|
|
585
702
|
prefillSemantics: 'not_applicable_training_workload',
|
|
586
703
|
});
|
|
704
|
+
const firstLoad = buildFirstLoadComposition({
|
|
705
|
+
modelLoadMs: timing.modelLoadMs,
|
|
706
|
+
firstTokenMs: timing.firstTokenMs,
|
|
707
|
+
firstResponseMs: timing.firstResponseMs,
|
|
708
|
+
});
|
|
587
709
|
return {
|
|
588
710
|
...trainingBench,
|
|
589
711
|
modelId: trainingBench.modelId || options.modelId || options.modelUrl || 'training',
|
|
@@ -600,6 +722,7 @@ async function runBenchSuite(options = {}) {
|
|
|
600
722
|
},
|
|
601
723
|
timing,
|
|
602
724
|
timingDiagnostics,
|
|
725
|
+
firstLoad,
|
|
603
726
|
output: null,
|
|
604
727
|
memoryStats: null,
|
|
605
728
|
deviceInfo: trainingBench.deviceInfo ?? resolveDeviceInfo(),
|
|
@@ -639,12 +762,22 @@ async function runBenchSuite(options = {}) {
|
|
|
639
762
|
};
|
|
640
763
|
}
|
|
641
764
|
|
|
642
|
-
const harness = await
|
|
765
|
+
const harness = await withHarnessPhase(
|
|
766
|
+
'bench.initializeSuiteModel',
|
|
767
|
+
{
|
|
768
|
+
modelId: options.modelId ?? null,
|
|
769
|
+
loadMode,
|
|
770
|
+
cacheMode,
|
|
771
|
+
},
|
|
772
|
+
() => initializeSuiteModel(options)
|
|
773
|
+
);
|
|
643
774
|
const benchRun = resolveBenchmarkRunSettings(runtimeConfig, harness.pipeline ?? harness);
|
|
644
775
|
const modelType = harness.manifest?.modelType || 'transformer';
|
|
645
|
-
|
|
776
|
+
const supportsEmbedding = modelSupportsEmbedding(harness.manifest);
|
|
777
|
+
if (options.expectedModelType === 'embedding' && !supportsEmbedding) {
|
|
646
778
|
throw new Error(
|
|
647
|
-
`Expected an embedding model for bench workload "${options.workload || 'inference'}", got "${modelType}"
|
|
779
|
+
`Expected an embedding-capable model for bench workload "${options.workload || 'inference'}", got modelType="${modelType}". ` +
|
|
780
|
+
`Set inference.supportsEmbedding=true in the manifest for text-generation models that should expose pipeline.embed().`
|
|
648
781
|
);
|
|
649
782
|
}
|
|
650
783
|
const safeModelLoadMs = toTimingNumber(harness.modelLoadMs, 0);
|
|
@@ -654,7 +787,7 @@ async function runBenchSuite(options = {}) {
|
|
|
654
787
|
let output = null;
|
|
655
788
|
let timing;
|
|
656
789
|
|
|
657
|
-
if (modelType === 'embedding') {
|
|
790
|
+
if (modelType === 'embedding' || (options.workload === 'embedding' && supportsEmbedding)) {
|
|
658
791
|
const durations = [];
|
|
659
792
|
const timedDurations = [];
|
|
660
793
|
const embeddingDims = [];
|
|
@@ -776,6 +909,8 @@ async function runBenchSuite(options = {}) {
|
|
|
776
909
|
const gpuDecodeSubmitWaitMs = [];
|
|
777
910
|
const gpuDecodeReadbackWaitMs = [];
|
|
778
911
|
const gpuDecodeOrchestrationMs = [];
|
|
912
|
+
const gpuPrefillRecordMs = [];
|
|
913
|
+
const gpuPrefillSubmitWaitMs = [];
|
|
779
914
|
const singleTokenSubmitWaitMs = [];
|
|
780
915
|
const singleTokenReadbackWaitMs = [];
|
|
781
916
|
const singleTokenOrchestrationMs = [];
|
|
@@ -795,10 +930,20 @@ async function runBenchSuite(options = {}) {
|
|
|
795
930
|
let lastBatchGuardReason = null;
|
|
796
931
|
for (let i = 0; i < warmupRuns + timedRuns; i++) {
|
|
797
932
|
harness.pipeline.reset?.();
|
|
798
|
-
const run = await
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
933
|
+
const run = await withHarnessPhase(
|
|
934
|
+
`bench.runTextInference[${i}]`,
|
|
935
|
+
{
|
|
936
|
+
modelId: options.modelId ?? harness.manifest?.modelId ?? null,
|
|
937
|
+
loadMode,
|
|
938
|
+
cacheMode,
|
|
939
|
+
warmupRuns,
|
|
940
|
+
timedRuns,
|
|
941
|
+
},
|
|
942
|
+
() => runTextInference(harness.pipeline, runtimeConfig, {
|
|
943
|
+
...benchRun,
|
|
944
|
+
...(options.inferenceInput ?? {}),
|
|
945
|
+
})
|
|
946
|
+
);
|
|
802
947
|
if (i === warmupRuns + timedRuns - 1) {
|
|
803
948
|
generatedText = run?.output ?? null;
|
|
804
949
|
generatedPromptInput = run?.promptInput ?? null;
|
|
@@ -830,6 +975,8 @@ async function runBenchSuite(options = {}) {
|
|
|
830
975
|
if (Number.isFinite(phaseGpu?.decodeRecordMs)) gpuDecodeRecordMs.push(phaseGpu.decodeRecordMs);
|
|
831
976
|
if (Number.isFinite(phaseGpu?.decodeSubmitWaitMs)) gpuDecodeSubmitWaitMs.push(phaseGpu.decodeSubmitWaitMs);
|
|
832
977
|
if (Number.isFinite(phaseGpu?.decodeReadbackWaitMs)) gpuDecodeReadbackWaitMs.push(phaseGpu.decodeReadbackWaitMs);
|
|
978
|
+
if (Number.isFinite(phaseGpu?.prefillRecordMs)) gpuPrefillRecordMs.push(phaseGpu.prefillRecordMs);
|
|
979
|
+
if (Number.isFinite(phaseGpu?.prefillSubmitWaitMs)) gpuPrefillSubmitWaitMs.push(phaseGpu.prefillSubmitWaitMs);
|
|
833
980
|
if (Number.isFinite(phaseGpu?.decodeOrchestrationMs)) {
|
|
834
981
|
gpuDecodeOrchestrationMs.push(phaseGpu.decodeOrchestrationMs);
|
|
835
982
|
}
|
|
@@ -873,6 +1020,8 @@ async function runBenchSuite(options = {}) {
|
|
|
873
1020
|
decodeSubmitWaitMs: computeSampleStats(gpuDecodeSubmitWaitMs),
|
|
874
1021
|
decodeReadbackWaitMs: computeSampleStats(gpuDecodeReadbackWaitMs),
|
|
875
1022
|
decodeOrchestrationMs: computeSampleStats(gpuDecodeOrchestrationMs),
|
|
1023
|
+
prefillRecordMs: computeSampleStats(gpuPrefillRecordMs),
|
|
1024
|
+
prefillSubmitWaitMs: computeSampleStats(gpuPrefillSubmitWaitMs),
|
|
876
1025
|
singleTokenSubmitWaitMs: computeSampleStats(singleTokenSubmitWaitMs),
|
|
877
1026
|
singleTokenReadbackWaitMs: computeSampleStats(singleTokenReadbackWaitMs),
|
|
878
1027
|
singleTokenOrchestrationMs: computeSampleStats(singleTokenOrchestrationMs),
|
|
@@ -1000,6 +1149,11 @@ async function runBenchSuite(options = {}) {
|
|
|
1000
1149
|
source: 'doppler',
|
|
1001
1150
|
prefillSemantics: 'internal_prefill_phase',
|
|
1002
1151
|
});
|
|
1152
|
+
const firstLoad = buildFirstLoadComposition({
|
|
1153
|
+
modelLoadMs: timing.modelLoadMs,
|
|
1154
|
+
firstTokenMs: timing.firstTokenMs,
|
|
1155
|
+
firstResponseMs: timing.firstResponseMs,
|
|
1156
|
+
});
|
|
1003
1157
|
const metricsWithContracts = buildSuiteContractMetrics('bench', metrics, harness.manifest);
|
|
1004
1158
|
return {
|
|
1005
1159
|
...summary,
|
|
@@ -1017,6 +1171,7 @@ async function runBenchSuite(options = {}) {
|
|
|
1017
1171
|
},
|
|
1018
1172
|
timing,
|
|
1019
1173
|
timingDiagnostics,
|
|
1174
|
+
firstLoad,
|
|
1020
1175
|
output,
|
|
1021
1176
|
metrics: metricsWithContracts,
|
|
1022
1177
|
memoryStats,
|
|
@@ -1072,6 +1227,35 @@ function shouldCaptureDebugSnapshot(mode, runtimeConfig) {
|
|
|
1072
1227
|
|| logLevel === 'verbose';
|
|
1073
1228
|
}
|
|
1074
1229
|
|
|
1230
|
+
function createHarnessPhaseError(error, phase, context = {}) {
|
|
1231
|
+
const message = error?.message || String(error);
|
|
1232
|
+
const wrapped = new Error(
|
|
1233
|
+
`Browser harness phase "${phase}" failed: ${message}`,
|
|
1234
|
+
error instanceof Error ? { cause: error } : undefined
|
|
1235
|
+
);
|
|
1236
|
+
wrapped.name = error?.name || 'Error';
|
|
1237
|
+
if (error?.code !== undefined) {
|
|
1238
|
+
wrapped.code = error.code;
|
|
1239
|
+
}
|
|
1240
|
+
wrapped.details = {
|
|
1241
|
+
...(error?.details && typeof error.details === 'object' ? error.details : {}),
|
|
1242
|
+
harnessPhase: phase,
|
|
1243
|
+
...context,
|
|
1244
|
+
};
|
|
1245
|
+
return wrapped;
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
async function withHarnessPhase(phase, context, run) {
|
|
1249
|
+
try {
|
|
1250
|
+
return await run();
|
|
1251
|
+
} catch (error) {
|
|
1252
|
+
if (error?.details?.harnessPhase) {
|
|
1253
|
+
throw error;
|
|
1254
|
+
}
|
|
1255
|
+
throw createHarnessPhaseError(error, phase, context);
|
|
1256
|
+
}
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1075
1259
|
export async function runBrowserSuite(options = {}) {
|
|
1076
1260
|
return runWithRuntimeIsolationForSuite(async () => {
|
|
1077
1261
|
const suiteTimestamp = resolveReportTimestamp(options.timestamp, 'runBrowserSuite timestamp');
|
|
@@ -1083,12 +1267,21 @@ export async function runBrowserSuite(options = {}) {
|
|
|
1083
1267
|
if (captureDebugSnapshot) {
|
|
1084
1268
|
clearLogHistory();
|
|
1085
1269
|
}
|
|
1086
|
-
const suiteResult = await
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1270
|
+
const suiteResult = await withHarnessPhase(
|
|
1271
|
+
'dispatchBrowserSuite',
|
|
1272
|
+
{
|
|
1273
|
+
mode,
|
|
1274
|
+
workload,
|
|
1275
|
+
modelId: options.modelId ?? null,
|
|
1276
|
+
loadMode: options.loadMode ?? null,
|
|
1277
|
+
},
|
|
1278
|
+
() => dispatchBrowserSuite(mode, workload, {
|
|
1279
|
+
...options,
|
|
1280
|
+
mode,
|
|
1281
|
+
workload,
|
|
1282
|
+
suite,
|
|
1283
|
+
})
|
|
1284
|
+
);
|
|
1092
1285
|
if (!suiteResult) {
|
|
1093
1286
|
throw createUnsupportedWorkloadError(workload, { ...harnessContext, mode });
|
|
1094
1287
|
}
|
|
@@ -42,6 +42,12 @@ export class KVCache {
|
|
|
42
42
|
totalTokensSeen: number;
|
|
43
43
|
memoryUsage: number;
|
|
44
44
|
gpuContext: GPUContext | null;
|
|
45
|
+
counters: {
|
|
46
|
+
updateCalls: number;
|
|
47
|
+
gpuUpdateCalls: number;
|
|
48
|
+
recordedGpuUpdateCalls: number;
|
|
49
|
+
tokensWritten: number;
|
|
50
|
+
};
|
|
45
51
|
|
|
46
52
|
/**
|
|
47
53
|
* @param config - KV cache configuration
|
|
@@ -63,6 +63,13 @@ export class KVCache {
|
|
|
63
63
|
|
|
64
64
|
this.totalTokensSeen = 0;
|
|
65
65
|
|
|
66
|
+
this.counters = {
|
|
67
|
+
updateCalls: 0,
|
|
68
|
+
gpuUpdateCalls: 0,
|
|
69
|
+
recordedGpuUpdateCalls: 0,
|
|
70
|
+
tokensWritten: 0,
|
|
71
|
+
};
|
|
72
|
+
|
|
66
73
|
// Memory usage tracking
|
|
67
74
|
|
|
68
75
|
this.memoryUsage = 0;
|
|
@@ -322,6 +329,8 @@ export class KVCache {
|
|
|
322
329
|
if (numNewTokens === 0) {
|
|
323
330
|
return;
|
|
324
331
|
}
|
|
332
|
+
this.counters.updateCalls += 1;
|
|
333
|
+
this.counters.tokensWritten += numNewTokens;
|
|
325
334
|
|
|
326
335
|
if (startPos + numNewTokens > this.maxSeqLen) {
|
|
327
336
|
throw new Error(
|
|
@@ -362,6 +371,8 @@ export class KVCache {
|
|
|
362
371
|
if (numTokens === 0) {
|
|
363
372
|
return;
|
|
364
373
|
}
|
|
374
|
+
this.counters.gpuUpdateCalls += 1;
|
|
375
|
+
this.counters.tokensWritten += numTokens;
|
|
365
376
|
|
|
366
377
|
const layer = (this.layers[layerIdx]);
|
|
367
378
|
const device = getDevice();
|
|
@@ -420,6 +431,8 @@ export class KVCache {
|
|
|
420
431
|
if (numTokens === 0) {
|
|
421
432
|
return;
|
|
422
433
|
}
|
|
434
|
+
this.counters.recordedGpuUpdateCalls += 1;
|
|
435
|
+
this.counters.tokensWritten += numTokens;
|
|
423
436
|
|
|
424
437
|
const encoder = recorder.getEncoder();
|
|
425
438
|
const layer = (this.layers[layerIdx]);
|
|
@@ -820,7 +833,9 @@ export class KVCache {
|
|
|
820
833
|
efficiency: used / actual,
|
|
821
834
|
seqLen: this.currentSeqLen,
|
|
822
835
|
maxSeqLen: this.maxSeqLen,
|
|
823
|
-
layout: this.layout
|
|
836
|
+
layout: this.layout,
|
|
837
|
+
kvDtype: this.kvDtype,
|
|
838
|
+
counters: { ...this.counters }
|
|
824
839
|
};
|
|
825
840
|
}
|
|
826
841
|
|
|
@@ -3,6 +3,7 @@ import { type GPUContext, type GPUBuffersResult, type MemoryStats } from './type
|
|
|
3
3
|
export interface MixedGeometryKVCacheConfig {
|
|
4
4
|
numLayers: number;
|
|
5
5
|
numHeads: number;
|
|
6
|
+
globalNumHeads?: number | null;
|
|
6
7
|
headDim: number;
|
|
7
8
|
globalHeadDim?: number | null;
|
|
8
9
|
maxSeqLen: number;
|
|
@@ -25,6 +25,9 @@ function buildLayerSpecs(config, bytesPerElem) {
|
|
|
25
25
|
? Math.trunc(config.globalHeadDim)
|
|
26
26
|
: baseHeadDim;
|
|
27
27
|
const numHeads = Number(config.numHeads);
|
|
28
|
+
const globalNumHeads = Number.isFinite(config.globalNumHeads) && config.globalNumHeads > 0
|
|
29
|
+
? Math.trunc(config.globalNumHeads)
|
|
30
|
+
: numHeads;
|
|
28
31
|
const maxSeqLen = Number(config.maxSeqLen);
|
|
29
32
|
const slidingWindow = Number.isFinite(config.slidingWindow) && config.slidingWindow > 0
|
|
30
33
|
? Math.min(Math.trunc(config.slidingWindow), maxSeqLen)
|
|
@@ -46,14 +49,15 @@ function buildLayerSpecs(config, bytesPerElem) {
|
|
|
46
49
|
const capacityTokens = isSliding && slidingWindow != null
|
|
47
50
|
? slidingWindow
|
|
48
51
|
: maxSeqLen;
|
|
49
|
-
const
|
|
52
|
+
const layerNumHeads = isSliding ? numHeads : globalNumHeads;
|
|
53
|
+
const kvSize = layerNumHeads * headDim;
|
|
50
54
|
const bytesPerToken = kvSize * bytesPerElem;
|
|
51
55
|
return {
|
|
52
56
|
layerIdx,
|
|
53
57
|
layerType,
|
|
54
58
|
layout: isSliding && slidingWindow != null ? 'ring' : 'contiguous',
|
|
55
59
|
headDim,
|
|
56
|
-
numHeads,
|
|
60
|
+
numHeads: layerNumHeads,
|
|
57
61
|
kvSize,
|
|
58
62
|
capacityTokens,
|
|
59
63
|
bytesPerToken,
|
|
@@ -120,6 +124,9 @@ export class MixedGeometryKVCache {
|
|
|
120
124
|
|
|
121
125
|
this.numLayers = Math.trunc(config.numLayers);
|
|
122
126
|
this.numHeads = Math.trunc(config.numHeads);
|
|
127
|
+
this.globalNumHeads = Number.isFinite(config.globalNumHeads) && config.globalNumHeads > 0
|
|
128
|
+
? Math.trunc(config.globalNumHeads)
|
|
129
|
+
: this.numHeads;
|
|
123
130
|
this.headDim = Math.max(
|
|
124
131
|
Math.trunc(config.headDim),
|
|
125
132
|
Number.isFinite(config.globalHeadDim) ? Math.trunc(config.globalHeadDim) : Math.trunc(config.headDim)
|
|
@@ -142,6 +149,7 @@ export class MixedGeometryKVCache {
|
|
|
142
149
|
...config,
|
|
143
150
|
numLayers: this.numLayers,
|
|
144
151
|
numHeads: this.numHeads,
|
|
152
|
+
globalNumHeads: this.globalNumHeads,
|
|
145
153
|
headDim: Math.trunc(config.headDim),
|
|
146
154
|
globalHeadDim: Number.isFinite(config.globalHeadDim) ? Math.trunc(config.globalHeadDim) : null,
|
|
147
155
|
maxSeqLen: this.maxSeqLen,
|
|
@@ -400,6 +408,8 @@ export class MixedGeometryKVCache {
|
|
|
400
408
|
seqLen: this.currentSeqLen,
|
|
401
409
|
maxSeqLen: this.maxSeqLen,
|
|
402
410
|
layout: this.layout,
|
|
411
|
+
kvDtype: this.kvDtype,
|
|
412
|
+
counters: null,
|
|
403
413
|
};
|
|
404
414
|
}
|
|
405
415
|
|
|
@@ -598,6 +598,8 @@ export class TieredKVCache {
|
|
|
598
598
|
seqLen: this.currentSeqLen,
|
|
599
599
|
maxSeqLen: this.maxSeqLen,
|
|
600
600
|
layout: 'paged',
|
|
601
|
+
kvDtype: this.coldDtype,
|
|
602
|
+
counters: null,
|
|
601
603
|
};
|
|
602
604
|
return {
|
|
603
605
|
theoretical: hotStats.theoretical + coldStats.theoretical,
|
|
@@ -607,6 +609,11 @@ export class TieredKVCache {
|
|
|
607
609
|
seqLen: this.currentSeqLen,
|
|
608
610
|
maxSeqLen: this.maxSeqLen,
|
|
609
611
|
layout: this.layout,
|
|
612
|
+
kvDtype: this.kvDtype,
|
|
613
|
+
counters: {
|
|
614
|
+
hot: hotStats.counters ?? null,
|
|
615
|
+
cold: coldStats.counters ?? null,
|
|
616
|
+
},
|
|
610
617
|
hot: hotStats,
|
|
611
618
|
cold: coldStats,
|
|
612
619
|
};
|
|
@@ -183,6 +183,8 @@ export interface MemoryStats {
|
|
|
183
183
|
seqLen: number;
|
|
184
184
|
maxSeqLen: number;
|
|
185
185
|
layout: 'contiguous' | 'contiguous_quantized' | 'paged' | 'tiered' | 'bdpa_paged';
|
|
186
|
+
kvDtype?: 'f16' | 'f32' | null;
|
|
187
|
+
counters?: Record<string, unknown> | null;
|
|
186
188
|
}
|
|
187
189
|
|
|
188
190
|
/**
|