doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
getActiveKernelPathPolicy,
|
|
12
12
|
} from '../config/kernel-path-loader.js';
|
|
13
13
|
import { validateTrainingMetricsReport } from '../config/schema/training-metrics.schema.js';
|
|
14
|
+
import { modelSupportsEmbedding } from '../config/schema/manifest.schema.js';
|
|
14
15
|
import {
|
|
15
16
|
resolveReportTimestamp,
|
|
16
17
|
resolveRuntime,
|
|
@@ -33,6 +34,7 @@ import {
|
|
|
33
34
|
normalizeWorkloadType,
|
|
34
35
|
assertDiffusionPerformanceArtifact,
|
|
35
36
|
toTimingNumber,
|
|
37
|
+
buildFirstLoadComposition,
|
|
36
38
|
safeToFixed,
|
|
37
39
|
sampleTimingNumber,
|
|
38
40
|
buildCanonicalTiming,
|
|
@@ -56,6 +58,8 @@ import {
|
|
|
56
58
|
runEnergySuite,
|
|
57
59
|
} from './browser-harness-diffusion-energy-suites.js';
|
|
58
60
|
import { collectTrainingArtifactsFromSuiteResult } from './browser-harness-report-helpers.js';
|
|
61
|
+
import { sha256Hex } from '../utils/sha256.js';
|
|
62
|
+
import { stableSortObject } from '../utils/stable-sort-object.js';
|
|
59
63
|
|
|
60
64
|
const TRAINING_SUITE_MODULE_PATH = '../experimental/training/suite.js';
|
|
61
65
|
let trainingSuiteModulePromise = null;
|
|
@@ -67,15 +71,6 @@ async function loadTrainingSuiteModule() {
|
|
|
67
71
|
return trainingSuiteModulePromise;
|
|
68
72
|
}
|
|
69
73
|
|
|
70
|
-
/**
|
|
71
|
-
* Clear the cached training suite module promise.
|
|
72
|
-
* Useful for testing or when the training module needs to be re-imported
|
|
73
|
-
* after configuration changes.
|
|
74
|
-
*/
|
|
75
|
-
export function clearTrainingSuiteModule() {
|
|
76
|
-
trainingSuiteModulePromise = null;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
74
|
export async function runTrainingSuite(options = {}) {
|
|
80
75
|
const module = await loadTrainingSuiteModule();
|
|
81
76
|
return module.runTrainingSuite(options);
|
|
@@ -95,48 +90,6 @@ async function runTrainingBenchSuite(options = {}) {
|
|
|
95
90
|
return module.runTrainingBenchSuite(options);
|
|
96
91
|
}
|
|
97
92
|
|
|
98
|
-
export async function initializeBrowserHarness(options = {}) {
|
|
99
|
-
const { modelUrl, onProgress, log } = options;
|
|
100
|
-
if (!modelUrl) {
|
|
101
|
-
throw new Error('modelUrl is required');
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
const runtime = resolveRuntime(options);
|
|
105
|
-
const result = await initializeInference(modelUrl, {
|
|
106
|
-
runtime,
|
|
107
|
-
onProgress,
|
|
108
|
-
log,
|
|
109
|
-
});
|
|
110
|
-
|
|
111
|
-
return { ...result, runtime };
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
export async function saveBrowserReport(modelId, report, options = {}) {
|
|
115
|
-
return saveReport(modelId, report, options);
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
export async function runBrowserHarness(options = {}) {
|
|
119
|
-
const harness = await initializeBrowserHarness(options);
|
|
120
|
-
const reportTimestamp = resolveReportTimestamp(options.timestamp, 'runBrowserHarness timestamp');
|
|
121
|
-
const modelId = options.modelId || harness.manifest?.modelId || 'unknown';
|
|
122
|
-
|
|
123
|
-
let report = options.report || null;
|
|
124
|
-
if (!report && typeof options.buildReport === 'function') {
|
|
125
|
-
report = await options.buildReport(harness);
|
|
126
|
-
}
|
|
127
|
-
if (!report) {
|
|
128
|
-
report = {
|
|
129
|
-
modelId,
|
|
130
|
-
timestamp: reportTimestamp,
|
|
131
|
-
};
|
|
132
|
-
} else if (!report.timestamp) {
|
|
133
|
-
report = { ...report, timestamp: reportTimestamp };
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
const reportInfo = await saveReport(modelId, report, { timestamp: report.timestamp || reportTimestamp });
|
|
137
|
-
return { ...harness, report, reportInfo };
|
|
138
|
-
}
|
|
139
|
-
|
|
140
93
|
const BROWSER_WORKLOAD_SET = Object.freeze([
|
|
141
94
|
'kernels',
|
|
142
95
|
'inference',
|
|
@@ -296,6 +249,147 @@ function resolveDispatchSuite(mode, workload) {
|
|
|
296
249
|
return workload;
|
|
297
250
|
}
|
|
298
251
|
|
|
252
|
+
function stableJson(value) {
|
|
253
|
+
return JSON.stringify(stableSortObject(value)) ?? 'null';
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function hashStableJson(value) {
|
|
257
|
+
return `sha256:${sha256Hex(stableJson(value))}`;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
export function resolveExecutionGraphHash(manifest) {
|
|
261
|
+
const execution = manifest?.inference?.execution;
|
|
262
|
+
if (!execution || typeof execution !== 'object') {
|
|
263
|
+
return null;
|
|
264
|
+
}
|
|
265
|
+
return hashStableJson(execution);
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function buildPerStepTokenProof(tokenIds) {
|
|
269
|
+
return tokenIds.map((tokenId, index) => ({
|
|
270
|
+
index,
|
|
271
|
+
tokenId,
|
|
272
|
+
tokenHash: hashStableJson({ index, tokenId }),
|
|
273
|
+
}));
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function buildKvCacheTranscriptSeed(kvCache, byteProof = null) {
|
|
277
|
+
const source = kvCache && typeof kvCache === 'object' ? kvCache : null;
|
|
278
|
+
const proof = byteProof && typeof byteProof === 'object' ? byteProof : null;
|
|
279
|
+
const seed = {
|
|
280
|
+
mode: proof ? 'stats+sha256-layer-kv-bytes' : (source ? 'stats' : 'not-captured'),
|
|
281
|
+
layout: typeof source?.layout === 'string' ? source.layout : null,
|
|
282
|
+
kvDtype: typeof source?.kvDtype === 'string' ? source.kvDtype : null,
|
|
283
|
+
seqLen: Number.isFinite(source?.seqLen) ? source.seqLen : null,
|
|
284
|
+
maxSeqLen: Number.isFinite(source?.maxSeqLen) ? source.maxSeqLen : null,
|
|
285
|
+
usedBytes: Number.isFinite(source?.usedBytes) ? source.usedBytes : null,
|
|
286
|
+
allocatedBytes: Number.isFinite(source?.allocatedBytes) ? source.allocatedBytes : null,
|
|
287
|
+
counters: source?.counters ?? null,
|
|
288
|
+
byteDigestMode: typeof proof?.mode === 'string' ? proof.mode : null,
|
|
289
|
+
byteDigest: typeof proof?.digest === 'string' ? proof.digest : null,
|
|
290
|
+
byteDigests: Array.isArray(proof?.layers) ? proof.layers : null,
|
|
291
|
+
};
|
|
292
|
+
return {
|
|
293
|
+
...seed,
|
|
294
|
+
stateHash: hashStableJson(seed),
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
export function buildReferenceTranscriptSeed(run, context = {}) {
|
|
299
|
+
const promptPayload = run.promptInput ?? run.prompt ?? null;
|
|
300
|
+
const outputText = typeof run.output === 'string' ? run.output : '';
|
|
301
|
+
const tokenIds = Array.isArray(run.tokenIds)
|
|
302
|
+
? run.tokenIds.map((value) => Number(value)).filter((value) => Number.isInteger(value))
|
|
303
|
+
: [];
|
|
304
|
+
const promptTokenIds = Array.isArray(run.promptTokenIds)
|
|
305
|
+
? run.promptTokenIds.map((value) => Number(value)).filter((value) => Number.isInteger(value))
|
|
306
|
+
: null;
|
|
307
|
+
const logitsDigests = Array.isArray(run.logitsDigests)
|
|
308
|
+
? run.logitsDigests
|
|
309
|
+
: [];
|
|
310
|
+
const hasCompleteLogitsDigests = logitsDigests.length === tokenIds.length && tokenIds.length > 0;
|
|
311
|
+
const transcript = {
|
|
312
|
+
schema: 'doppler.reference-transcript/v1',
|
|
313
|
+
source: {
|
|
314
|
+
kind: 'inline-browser-suite',
|
|
315
|
+
path: 'inline',
|
|
316
|
+
hash: 'sha256:' + '0'.repeat(64),
|
|
317
|
+
},
|
|
318
|
+
executionGraphHash: context.executionGraphHash ?? null,
|
|
319
|
+
surface: 'browser-webgpu',
|
|
320
|
+
prompt: {
|
|
321
|
+
identity: typeof run.prompt === 'string' && run.prompt.trim() ? run.prompt : 'promptInput',
|
|
322
|
+
hash: hashStableJson(promptPayload),
|
|
323
|
+
tokenIdsHash: promptTokenIds ? hashStableJson(promptTokenIds) : null,
|
|
324
|
+
tokenCount: promptTokenIds ? promptTokenIds.length : null,
|
|
325
|
+
},
|
|
326
|
+
output: {
|
|
327
|
+
textHash: `sha256:${sha256Hex(outputText)}`,
|
|
328
|
+
tokensGenerated: tokenIds.length,
|
|
329
|
+
stopReason: typeof run.phase?.stopReason === 'string' ? run.phase.stopReason : 'unknown',
|
|
330
|
+
stopTokenId: Number.isInteger(run.phase?.stopTokenId) ? run.phase.stopTokenId : null,
|
|
331
|
+
},
|
|
332
|
+
tokens: {
|
|
333
|
+
ids: tokenIds,
|
|
334
|
+
generatedTokenIdsHash: hashStableJson(tokenIds),
|
|
335
|
+
generatedTextHash: `sha256:${sha256Hex(outputText)}`,
|
|
336
|
+
preview: Array.isArray(run.tokenDiagnostics?.preview) ? run.tokenDiagnostics.preview : [],
|
|
337
|
+
perStep: buildPerStepTokenProof(tokenIds),
|
|
338
|
+
coverage: {
|
|
339
|
+
mode: 'full-token-ids',
|
|
340
|
+
omitted: 0,
|
|
341
|
+
},
|
|
342
|
+
},
|
|
343
|
+
phase: {
|
|
344
|
+
prefillMs: Number.isFinite(run.phase?.prefillMs) ? run.phase.prefillMs : null,
|
|
345
|
+
decodeMs: Number.isFinite(run.phase?.decodeMs) ? run.phase.decodeMs : null,
|
|
346
|
+
prefillTokens: Number.isFinite(run.phase?.prefillTokens) ? run.phase.prefillTokens : null,
|
|
347
|
+
decodeTokens: Number.isFinite(run.phase?.decodeTokens) ? run.phase.decodeTokens : null,
|
|
348
|
+
},
|
|
349
|
+
kvCache: buildKvCacheTranscriptSeed(
|
|
350
|
+
run.phase?.kvCache ?? context.kvCache ?? null,
|
|
351
|
+
run.kvCacheByteProof ?? null
|
|
352
|
+
),
|
|
353
|
+
logits: hasCompleteLogitsDigests ? {
|
|
354
|
+
mode: 'sha256-per-step',
|
|
355
|
+
perStepDigests: logitsDigests.map((entry) => entry.digest),
|
|
356
|
+
steps: logitsDigests,
|
|
357
|
+
} : {
|
|
358
|
+
mode: 'not-captured',
|
|
359
|
+
reason: logitsDigests.length > 0
|
|
360
|
+
? 'Per-step logits digest count did not match generated token count.'
|
|
361
|
+
: 'Per-step logits digests were not requested for this browser harness run.',
|
|
362
|
+
perStepDigests: null,
|
|
363
|
+
},
|
|
364
|
+
tolerance: {
|
|
365
|
+
tokenPolicy: 'exact generated token IDs',
|
|
366
|
+
logitsPolicy: hasCompleteLogitsDigests
|
|
367
|
+
? 'exact sha256 digest per generated step over finalized f32 logits before sampling'
|
|
368
|
+
: 'not captured',
|
|
369
|
+
kvPolicy: run.kvCacheByteProof
|
|
370
|
+
? 'exact sha256 digest over used KV cache bytes by layer/key/value'
|
|
371
|
+
: 'metadata hash only; KV tensor bytes are not read back by default',
|
|
372
|
+
},
|
|
373
|
+
};
|
|
374
|
+
return {
|
|
375
|
+
...transcript,
|
|
376
|
+
source: {
|
|
377
|
+
...transcript.source,
|
|
378
|
+
hash: hashStableJson({
|
|
379
|
+
prompt: transcript.prompt,
|
|
380
|
+
output: transcript.output,
|
|
381
|
+
tokens: {
|
|
382
|
+
generatedTokenIdsHash: transcript.tokens.generatedTokenIdsHash,
|
|
383
|
+
generatedTextHash: transcript.tokens.generatedTextHash,
|
|
384
|
+
},
|
|
385
|
+
phase: transcript.phase,
|
|
386
|
+
kvCache: transcript.kvCache,
|
|
387
|
+
logits: transcript.logits,
|
|
388
|
+
}),
|
|
389
|
+
},
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
|
|
299
393
|
async function runKernelSuite(options = {}) {
|
|
300
394
|
const startTime = performance.now();
|
|
301
395
|
const { testHarness, initGPU } = await import('../../tests/kernels/browser/test-page.js');
|
|
@@ -324,23 +418,33 @@ async function runKernelSuite(options = {}) {
|
|
|
324
418
|
|
|
325
419
|
async function runInferenceSuite(options = {}) {
|
|
326
420
|
const startTime = performance.now();
|
|
327
|
-
const
|
|
421
|
+
const cacheMode = normalizeCacheMode(options.cacheMode);
|
|
422
|
+
const loadMode = normalizeLoadMode(options.loadMode, !!options.modelUrl, options.modelUrl);
|
|
423
|
+
const harness = await withHarnessPhase(
|
|
424
|
+
'inference.initializeSuiteModel',
|
|
425
|
+
{
|
|
426
|
+
modelId: options.modelId ?? null,
|
|
427
|
+
loadMode,
|
|
428
|
+
cacheMode,
|
|
429
|
+
},
|
|
430
|
+
() => initializeSuiteModel(options)
|
|
431
|
+
);
|
|
328
432
|
const runtimeConfig = getRuntimeConfig();
|
|
329
433
|
const modelType = harness.manifest?.modelType || 'transformer';
|
|
330
|
-
|
|
434
|
+
const supportsEmbedding = modelSupportsEmbedding(harness.manifest);
|
|
435
|
+
if (options.expectedModelType === 'embedding' && !supportsEmbedding) {
|
|
331
436
|
throw new Error(
|
|
332
|
-
`Expected an embedding model for workload "${options.workload || 'inference'}", got "${modelType}"
|
|
437
|
+
`Expected an embedding-capable model for workload "${options.workload || 'inference'}", got modelType="${modelType}". ` +
|
|
438
|
+
`Set inference.supportsEmbedding=true in the manifest for text-generation models that should expose pipeline.embed().`
|
|
333
439
|
);
|
|
334
440
|
}
|
|
335
|
-
const cacheMode = normalizeCacheMode(options.cacheMode);
|
|
336
|
-
const loadMode = normalizeLoadMode(options.loadMode, !!options.modelUrl, options.modelUrl);
|
|
337
441
|
const safeModelLoadMs = toTimingNumber(harness.modelLoadMs, 0);
|
|
338
442
|
|
|
339
443
|
let results;
|
|
340
444
|
let output = null;
|
|
341
445
|
let metrics;
|
|
342
446
|
|
|
343
|
-
if (modelType === 'embedding') {
|
|
447
|
+
if (modelType === 'embedding' || (options.workload === 'embedding' && supportsEmbedding)) {
|
|
344
448
|
const run = await runEmbedding(harness.pipeline, runtimeConfig);
|
|
345
449
|
const semantic = await runEmbeddingSemanticChecks(harness.pipeline, options);
|
|
346
450
|
const isValidEmbedding = run.embeddingDim > 0 && run.nonFiniteCount === 0;
|
|
@@ -464,6 +568,8 @@ async function runInferenceSuite(options = {}) {
|
|
|
464
568
|
decodeMs: safeToFixed(run.phase.decodeMs),
|
|
465
569
|
prefillTokens: Math.round(run.phase.prefillTokens),
|
|
466
570
|
decodeTokens: Math.round(run.phase.decodeTokens),
|
|
571
|
+
stopReason: run.phase.stopReason ?? null,
|
|
572
|
+
stopTokenId: Number.isInteger(run.phase.stopTokenId) ? run.phase.stopTokenId : null,
|
|
467
573
|
prefillTokensPerSec: safeToFixed(run.phase.prefillTokensPerSec),
|
|
468
574
|
prefillTokensPerSecTtft: safeToFixed(run.phase.prefillTokensPerSecTtft),
|
|
469
575
|
decodeTokensPerSec: safeToFixed(run.phase.decodeTokensPerSec),
|
|
@@ -476,6 +582,11 @@ async function runInferenceSuite(options = {}) {
|
|
|
476
582
|
kernelPathId: run.phase.kernelPathId,
|
|
477
583
|
kernelPathSource: run.phase.kernelPathSource,
|
|
478
584
|
generationDiagnostics: run.tokenDiagnostics,
|
|
585
|
+
kvCache: run.phase.kvCache ?? null,
|
|
586
|
+
referenceTranscript: buildReferenceTranscriptSeed(run, {
|
|
587
|
+
executionGraphHash: resolveExecutionGraphHash(harness.manifest),
|
|
588
|
+
kvCache: run.phase.kvCache ?? null,
|
|
589
|
+
}),
|
|
479
590
|
operatorDiagnostics: run.phase.operatorDiagnostics ?? null,
|
|
480
591
|
};
|
|
481
592
|
}
|
|
@@ -509,6 +620,11 @@ async function runInferenceSuite(options = {}) {
|
|
|
509
620
|
source: 'doppler',
|
|
510
621
|
prefillSemantics: 'internal_prefill_phase',
|
|
511
622
|
});
|
|
623
|
+
const firstLoad = buildFirstLoadComposition({
|
|
624
|
+
modelLoadMs: timing.modelLoadMs,
|
|
625
|
+
firstTokenMs: timing.firstTokenMs,
|
|
626
|
+
firstResponseMs: timing.firstResponseMs,
|
|
627
|
+
});
|
|
512
628
|
const metricsWithContracts = buildSuiteContractMetrics(
|
|
513
629
|
options.suiteName || 'inference',
|
|
514
630
|
metrics,
|
|
@@ -530,6 +646,7 @@ async function runInferenceSuite(options = {}) {
|
|
|
530
646
|
},
|
|
531
647
|
timing,
|
|
532
648
|
timingDiagnostics,
|
|
649
|
+
firstLoad,
|
|
533
650
|
output,
|
|
534
651
|
metrics: metricsWithContracts,
|
|
535
652
|
memoryStats,
|
|
@@ -584,6 +701,11 @@ async function runBenchSuite(options = {}) {
|
|
|
584
701
|
source: 'doppler',
|
|
585
702
|
prefillSemantics: 'not_applicable_training_workload',
|
|
586
703
|
});
|
|
704
|
+
const firstLoad = buildFirstLoadComposition({
|
|
705
|
+
modelLoadMs: timing.modelLoadMs,
|
|
706
|
+
firstTokenMs: timing.firstTokenMs,
|
|
707
|
+
firstResponseMs: timing.firstResponseMs,
|
|
708
|
+
});
|
|
587
709
|
return {
|
|
588
710
|
...trainingBench,
|
|
589
711
|
modelId: trainingBench.modelId || options.modelId || options.modelUrl || 'training',
|
|
@@ -600,6 +722,7 @@ async function runBenchSuite(options = {}) {
|
|
|
600
722
|
},
|
|
601
723
|
timing,
|
|
602
724
|
timingDiagnostics,
|
|
725
|
+
firstLoad,
|
|
603
726
|
output: null,
|
|
604
727
|
memoryStats: null,
|
|
605
728
|
deviceInfo: trainingBench.deviceInfo ?? resolveDeviceInfo(),
|
|
@@ -639,12 +762,22 @@ async function runBenchSuite(options = {}) {
|
|
|
639
762
|
};
|
|
640
763
|
}
|
|
641
764
|
|
|
642
|
-
const harness = await
|
|
765
|
+
const harness = await withHarnessPhase(
|
|
766
|
+
'bench.initializeSuiteModel',
|
|
767
|
+
{
|
|
768
|
+
modelId: options.modelId ?? null,
|
|
769
|
+
loadMode,
|
|
770
|
+
cacheMode,
|
|
771
|
+
},
|
|
772
|
+
() => initializeSuiteModel(options)
|
|
773
|
+
);
|
|
643
774
|
const benchRun = resolveBenchmarkRunSettings(runtimeConfig, harness.pipeline ?? harness);
|
|
644
775
|
const modelType = harness.manifest?.modelType || 'transformer';
|
|
645
|
-
|
|
776
|
+
const supportsEmbedding = modelSupportsEmbedding(harness.manifest);
|
|
777
|
+
if (options.expectedModelType === 'embedding' && !supportsEmbedding) {
|
|
646
778
|
throw new Error(
|
|
647
|
-
`Expected an embedding model for bench workload "${options.workload || 'inference'}", got "${modelType}"
|
|
779
|
+
`Expected an embedding-capable model for bench workload "${options.workload || 'inference'}", got modelType="${modelType}". ` +
|
|
780
|
+
`Set inference.supportsEmbedding=true in the manifest for text-generation models that should expose pipeline.embed().`
|
|
648
781
|
);
|
|
649
782
|
}
|
|
650
783
|
const safeModelLoadMs = toTimingNumber(harness.modelLoadMs, 0);
|
|
@@ -654,7 +787,7 @@ async function runBenchSuite(options = {}) {
|
|
|
654
787
|
let output = null;
|
|
655
788
|
let timing;
|
|
656
789
|
|
|
657
|
-
if (modelType === 'embedding') {
|
|
790
|
+
if (modelType === 'embedding' || (options.workload === 'embedding' && supportsEmbedding)) {
|
|
658
791
|
const durations = [];
|
|
659
792
|
const timedDurations = [];
|
|
660
793
|
const embeddingDims = [];
|
|
@@ -775,6 +908,9 @@ async function runBenchSuite(options = {}) {
|
|
|
775
908
|
const gpuDecodeRecordMs = [];
|
|
776
909
|
const gpuDecodeSubmitWaitMs = [];
|
|
777
910
|
const gpuDecodeReadbackWaitMs = [];
|
|
911
|
+
const gpuDecodeOrchestrationMs = [];
|
|
912
|
+
const gpuPrefillRecordMs = [];
|
|
913
|
+
const gpuPrefillSubmitWaitMs = [];
|
|
778
914
|
const singleTokenSubmitWaitMs = [];
|
|
779
915
|
const singleTokenReadbackWaitMs = [];
|
|
780
916
|
const singleTokenOrchestrationMs = [];
|
|
@@ -794,10 +930,20 @@ async function runBenchSuite(options = {}) {
|
|
|
794
930
|
let lastBatchGuardReason = null;
|
|
795
931
|
for (let i = 0; i < warmupRuns + timedRuns; i++) {
|
|
796
932
|
harness.pipeline.reset?.();
|
|
797
|
-
const run = await
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
933
|
+
const run = await withHarnessPhase(
|
|
934
|
+
`bench.runTextInference[${i}]`,
|
|
935
|
+
{
|
|
936
|
+
modelId: options.modelId ?? harness.manifest?.modelId ?? null,
|
|
937
|
+
loadMode,
|
|
938
|
+
cacheMode,
|
|
939
|
+
warmupRuns,
|
|
940
|
+
timedRuns,
|
|
941
|
+
},
|
|
942
|
+
() => runTextInference(harness.pipeline, runtimeConfig, {
|
|
943
|
+
...benchRun,
|
|
944
|
+
...(options.inferenceInput ?? {}),
|
|
945
|
+
})
|
|
946
|
+
);
|
|
801
947
|
if (i === warmupRuns + timedRuns - 1) {
|
|
802
948
|
generatedText = run?.output ?? null;
|
|
803
949
|
generatedPromptInput = run?.promptInput ?? null;
|
|
@@ -829,6 +975,11 @@ async function runBenchSuite(options = {}) {
|
|
|
829
975
|
if (Number.isFinite(phaseGpu?.decodeRecordMs)) gpuDecodeRecordMs.push(phaseGpu.decodeRecordMs);
|
|
830
976
|
if (Number.isFinite(phaseGpu?.decodeSubmitWaitMs)) gpuDecodeSubmitWaitMs.push(phaseGpu.decodeSubmitWaitMs);
|
|
831
977
|
if (Number.isFinite(phaseGpu?.decodeReadbackWaitMs)) gpuDecodeReadbackWaitMs.push(phaseGpu.decodeReadbackWaitMs);
|
|
978
|
+
if (Number.isFinite(phaseGpu?.prefillRecordMs)) gpuPrefillRecordMs.push(phaseGpu.prefillRecordMs);
|
|
979
|
+
if (Number.isFinite(phaseGpu?.prefillSubmitWaitMs)) gpuPrefillSubmitWaitMs.push(phaseGpu.prefillSubmitWaitMs);
|
|
980
|
+
if (Number.isFinite(phaseGpu?.decodeOrchestrationMs)) {
|
|
981
|
+
gpuDecodeOrchestrationMs.push(phaseGpu.decodeOrchestrationMs);
|
|
982
|
+
}
|
|
832
983
|
if (Number.isFinite(phaseGpu?.singleTokenSubmitWaitMs)) singleTokenSubmitWaitMs.push(phaseGpu.singleTokenSubmitWaitMs);
|
|
833
984
|
if (Number.isFinite(phaseGpu?.singleTokenReadbackWaitMs)) singleTokenReadbackWaitMs.push(phaseGpu.singleTokenReadbackWaitMs);
|
|
834
985
|
if (Number.isFinite(phaseGpu?.singleTokenOrchestrationMs)) singleTokenOrchestrationMs.push(phaseGpu.singleTokenOrchestrationMs);
|
|
@@ -858,6 +1009,7 @@ async function runBenchSuite(options = {}) {
|
|
|
858
1009
|
const decodeTokensStats = computeSampleStats(decodeTokens);
|
|
859
1010
|
const hasGpuStats = gpuPrefillMs.length > 0 || gpuDecodeMs.length > 0 || gpuDecodeRecordMs.length > 0
|
|
860
1011
|
|| gpuDecodeSubmitWaitMs.length > 0 || gpuDecodeReadbackWaitMs.length > 0
|
|
1012
|
+
|| gpuDecodeOrchestrationMs.length > 0
|
|
861
1013
|
|| singleTokenSubmitWaitMs.length > 0 || singleTokenReadbackWaitMs.length > 0
|
|
862
1014
|
|| singleTokenOrchestrationMs.length > 0;
|
|
863
1015
|
const gpuPhaseStats = hasGpuStats
|
|
@@ -867,6 +1019,9 @@ async function runBenchSuite(options = {}) {
|
|
|
867
1019
|
decodeRecordMs: computeSampleStats(gpuDecodeRecordMs),
|
|
868
1020
|
decodeSubmitWaitMs: computeSampleStats(gpuDecodeSubmitWaitMs),
|
|
869
1021
|
decodeReadbackWaitMs: computeSampleStats(gpuDecodeReadbackWaitMs),
|
|
1022
|
+
decodeOrchestrationMs: computeSampleStats(gpuDecodeOrchestrationMs),
|
|
1023
|
+
prefillRecordMs: computeSampleStats(gpuPrefillRecordMs),
|
|
1024
|
+
prefillSubmitWaitMs: computeSampleStats(gpuPrefillSubmitWaitMs),
|
|
870
1025
|
singleTokenSubmitWaitMs: computeSampleStats(singleTokenSubmitWaitMs),
|
|
871
1026
|
singleTokenReadbackWaitMs: computeSampleStats(singleTokenReadbackWaitMs),
|
|
872
1027
|
singleTokenOrchestrationMs: computeSampleStats(singleTokenOrchestrationMs),
|
|
@@ -913,6 +1068,7 @@ async function runBenchSuite(options = {}) {
|
|
|
913
1068
|
metrics = {
|
|
914
1069
|
warmupRuns,
|
|
915
1070
|
timedRuns,
|
|
1071
|
+
...(Number.isFinite(benchRun?.seed) ? { seed: benchRun.seed } : {}),
|
|
916
1072
|
prompt: benchRun.promptLabel,
|
|
917
1073
|
maxTokens: benchRun.maxTokens,
|
|
918
1074
|
decodeTokensPerSec: sampleTimingNumber(decodeTokensPerSecStats, 'median'),
|
|
@@ -993,6 +1149,11 @@ async function runBenchSuite(options = {}) {
|
|
|
993
1149
|
source: 'doppler',
|
|
994
1150
|
prefillSemantics: 'internal_prefill_phase',
|
|
995
1151
|
});
|
|
1152
|
+
const firstLoad = buildFirstLoadComposition({
|
|
1153
|
+
modelLoadMs: timing.modelLoadMs,
|
|
1154
|
+
firstTokenMs: timing.firstTokenMs,
|
|
1155
|
+
firstResponseMs: timing.firstResponseMs,
|
|
1156
|
+
});
|
|
996
1157
|
const metricsWithContracts = buildSuiteContractMetrics('bench', metrics, harness.manifest);
|
|
997
1158
|
return {
|
|
998
1159
|
...summary,
|
|
@@ -1010,6 +1171,7 @@ async function runBenchSuite(options = {}) {
|
|
|
1010
1171
|
},
|
|
1011
1172
|
timing,
|
|
1012
1173
|
timingDiagnostics,
|
|
1174
|
+
firstLoad,
|
|
1013
1175
|
output,
|
|
1014
1176
|
metrics: metricsWithContracts,
|
|
1015
1177
|
memoryStats,
|
|
@@ -1065,6 +1227,35 @@ function shouldCaptureDebugSnapshot(mode, runtimeConfig) {
|
|
|
1065
1227
|
|| logLevel === 'verbose';
|
|
1066
1228
|
}
|
|
1067
1229
|
|
|
1230
|
+
function createHarnessPhaseError(error, phase, context = {}) {
|
|
1231
|
+
const message = error?.message || String(error);
|
|
1232
|
+
const wrapped = new Error(
|
|
1233
|
+
`Browser harness phase "${phase}" failed: ${message}`,
|
|
1234
|
+
error instanceof Error ? { cause: error } : undefined
|
|
1235
|
+
);
|
|
1236
|
+
wrapped.name = error?.name || 'Error';
|
|
1237
|
+
if (error?.code !== undefined) {
|
|
1238
|
+
wrapped.code = error.code;
|
|
1239
|
+
}
|
|
1240
|
+
wrapped.details = {
|
|
1241
|
+
...(error?.details && typeof error.details === 'object' ? error.details : {}),
|
|
1242
|
+
harnessPhase: phase,
|
|
1243
|
+
...context,
|
|
1244
|
+
};
|
|
1245
|
+
return wrapped;
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
async function withHarnessPhase(phase, context, run) {
|
|
1249
|
+
try {
|
|
1250
|
+
return await run();
|
|
1251
|
+
} catch (error) {
|
|
1252
|
+
if (error?.details?.harnessPhase) {
|
|
1253
|
+
throw error;
|
|
1254
|
+
}
|
|
1255
|
+
throw createHarnessPhaseError(error, phase, context);
|
|
1256
|
+
}
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1068
1259
|
export async function runBrowserSuite(options = {}) {
|
|
1069
1260
|
return runWithRuntimeIsolationForSuite(async () => {
|
|
1070
1261
|
const suiteTimestamp = resolveReportTimestamp(options.timestamp, 'runBrowserSuite timestamp');
|
|
@@ -1076,12 +1267,21 @@ export async function runBrowserSuite(options = {}) {
|
|
|
1076
1267
|
if (captureDebugSnapshot) {
|
|
1077
1268
|
clearLogHistory();
|
|
1078
1269
|
}
|
|
1079
|
-
const suiteResult = await
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1270
|
+
const suiteResult = await withHarnessPhase(
|
|
1271
|
+
'dispatchBrowserSuite',
|
|
1272
|
+
{
|
|
1273
|
+
mode,
|
|
1274
|
+
workload,
|
|
1275
|
+
modelId: options.modelId ?? null,
|
|
1276
|
+
loadMode: options.loadMode ?? null,
|
|
1277
|
+
},
|
|
1278
|
+
() => dispatchBrowserSuite(mode, workload, {
|
|
1279
|
+
...options,
|
|
1280
|
+
mode,
|
|
1281
|
+
workload,
|
|
1282
|
+
suite,
|
|
1283
|
+
})
|
|
1284
|
+
);
|
|
1085
1285
|
if (!suiteResult) {
|
|
1086
1286
|
throw createUnsupportedWorkloadError(workload, { ...harnessContext, mode });
|
|
1087
1287
|
}
|
|
@@ -42,6 +42,12 @@ export class KVCache {
|
|
|
42
42
|
totalTokensSeen: number;
|
|
43
43
|
memoryUsage: number;
|
|
44
44
|
gpuContext: GPUContext | null;
|
|
45
|
+
counters: {
|
|
46
|
+
updateCalls: number;
|
|
47
|
+
gpuUpdateCalls: number;
|
|
48
|
+
recordedGpuUpdateCalls: number;
|
|
49
|
+
tokensWritten: number;
|
|
50
|
+
};
|
|
45
51
|
|
|
46
52
|
/**
|
|
47
53
|
* @param config - KV cache configuration
|
|
@@ -63,6 +63,13 @@ export class KVCache {
|
|
|
63
63
|
|
|
64
64
|
this.totalTokensSeen = 0;
|
|
65
65
|
|
|
66
|
+
this.counters = {
|
|
67
|
+
updateCalls: 0,
|
|
68
|
+
gpuUpdateCalls: 0,
|
|
69
|
+
recordedGpuUpdateCalls: 0,
|
|
70
|
+
tokensWritten: 0,
|
|
71
|
+
};
|
|
72
|
+
|
|
66
73
|
// Memory usage tracking
|
|
67
74
|
|
|
68
75
|
this.memoryUsage = 0;
|
|
@@ -322,6 +329,8 @@ export class KVCache {
|
|
|
322
329
|
if (numNewTokens === 0) {
|
|
323
330
|
return;
|
|
324
331
|
}
|
|
332
|
+
this.counters.updateCalls += 1;
|
|
333
|
+
this.counters.tokensWritten += numNewTokens;
|
|
325
334
|
|
|
326
335
|
if (startPos + numNewTokens > this.maxSeqLen) {
|
|
327
336
|
throw new Error(
|
|
@@ -362,6 +371,8 @@ export class KVCache {
|
|
|
362
371
|
if (numTokens === 0) {
|
|
363
372
|
return;
|
|
364
373
|
}
|
|
374
|
+
this.counters.gpuUpdateCalls += 1;
|
|
375
|
+
this.counters.tokensWritten += numTokens;
|
|
365
376
|
|
|
366
377
|
const layer = (this.layers[layerIdx]);
|
|
367
378
|
const device = getDevice();
|
|
@@ -420,6 +431,8 @@ export class KVCache {
|
|
|
420
431
|
if (numTokens === 0) {
|
|
421
432
|
return;
|
|
422
433
|
}
|
|
434
|
+
this.counters.recordedGpuUpdateCalls += 1;
|
|
435
|
+
this.counters.tokensWritten += numTokens;
|
|
423
436
|
|
|
424
437
|
const encoder = recorder.getEncoder();
|
|
425
438
|
const layer = (this.layers[layerIdx]);
|
|
@@ -820,7 +833,9 @@ export class KVCache {
|
|
|
820
833
|
efficiency: used / actual,
|
|
821
834
|
seqLen: this.currentSeqLen,
|
|
822
835
|
maxSeqLen: this.maxSeqLen,
|
|
823
|
-
layout: this.layout
|
|
836
|
+
layout: this.layout,
|
|
837
|
+
kvDtype: this.kvDtype,
|
|
838
|
+
counters: { ...this.counters }
|
|
824
839
|
};
|
|
825
840
|
}
|
|
826
841
|
|
|
@@ -3,6 +3,7 @@ import { type GPUContext, type GPUBuffersResult, type MemoryStats } from './type
|
|
|
3
3
|
export interface MixedGeometryKVCacheConfig {
|
|
4
4
|
numLayers: number;
|
|
5
5
|
numHeads: number;
|
|
6
|
+
globalNumHeads?: number | null;
|
|
6
7
|
headDim: number;
|
|
7
8
|
globalHeadDim?: number | null;
|
|
8
9
|
maxSeqLen: number;
|
|
@@ -25,6 +25,9 @@ function buildLayerSpecs(config, bytesPerElem) {
|
|
|
25
25
|
? Math.trunc(config.globalHeadDim)
|
|
26
26
|
: baseHeadDim;
|
|
27
27
|
const numHeads = Number(config.numHeads);
|
|
28
|
+
const globalNumHeads = Number.isFinite(config.globalNumHeads) && config.globalNumHeads > 0
|
|
29
|
+
? Math.trunc(config.globalNumHeads)
|
|
30
|
+
: numHeads;
|
|
28
31
|
const maxSeqLen = Number(config.maxSeqLen);
|
|
29
32
|
const slidingWindow = Number.isFinite(config.slidingWindow) && config.slidingWindow > 0
|
|
30
33
|
? Math.min(Math.trunc(config.slidingWindow), maxSeqLen)
|
|
@@ -46,14 +49,15 @@ function buildLayerSpecs(config, bytesPerElem) {
|
|
|
46
49
|
const capacityTokens = isSliding && slidingWindow != null
|
|
47
50
|
? slidingWindow
|
|
48
51
|
: maxSeqLen;
|
|
49
|
-
const
|
|
52
|
+
const layerNumHeads = isSliding ? numHeads : globalNumHeads;
|
|
53
|
+
const kvSize = layerNumHeads * headDim;
|
|
50
54
|
const bytesPerToken = kvSize * bytesPerElem;
|
|
51
55
|
return {
|
|
52
56
|
layerIdx,
|
|
53
57
|
layerType,
|
|
54
58
|
layout: isSliding && slidingWindow != null ? 'ring' : 'contiguous',
|
|
55
59
|
headDim,
|
|
56
|
-
numHeads,
|
|
60
|
+
numHeads: layerNumHeads,
|
|
57
61
|
kvSize,
|
|
58
62
|
capacityTokens,
|
|
59
63
|
bytesPerToken,
|
|
@@ -120,6 +124,9 @@ export class MixedGeometryKVCache {
|
|
|
120
124
|
|
|
121
125
|
this.numLayers = Math.trunc(config.numLayers);
|
|
122
126
|
this.numHeads = Math.trunc(config.numHeads);
|
|
127
|
+
this.globalNumHeads = Number.isFinite(config.globalNumHeads) && config.globalNumHeads > 0
|
|
128
|
+
? Math.trunc(config.globalNumHeads)
|
|
129
|
+
: this.numHeads;
|
|
123
130
|
this.headDim = Math.max(
|
|
124
131
|
Math.trunc(config.headDim),
|
|
125
132
|
Number.isFinite(config.globalHeadDim) ? Math.trunc(config.globalHeadDim) : Math.trunc(config.headDim)
|
|
@@ -142,6 +149,7 @@ export class MixedGeometryKVCache {
|
|
|
142
149
|
...config,
|
|
143
150
|
numLayers: this.numLayers,
|
|
144
151
|
numHeads: this.numHeads,
|
|
152
|
+
globalNumHeads: this.globalNumHeads,
|
|
145
153
|
headDim: Math.trunc(config.headDim),
|
|
146
154
|
globalHeadDim: Number.isFinite(config.globalHeadDim) ? Math.trunc(config.globalHeadDim) : null,
|
|
147
155
|
maxSeqLen: this.maxSeqLen,
|
|
@@ -400,6 +408,8 @@ export class MixedGeometryKVCache {
|
|
|
400
408
|
seqLen: this.currentSeqLen,
|
|
401
409
|
maxSeqLen: this.maxSeqLen,
|
|
402
410
|
layout: this.layout,
|
|
411
|
+
kvDtype: this.kvDtype,
|
|
412
|
+
counters: null,
|
|
403
413
|
};
|
|
404
414
|
}
|
|
405
415
|
|
|
@@ -598,6 +598,8 @@ export class TieredKVCache {
|
|
|
598
598
|
seqLen: this.currentSeqLen,
|
|
599
599
|
maxSeqLen: this.maxSeqLen,
|
|
600
600
|
layout: 'paged',
|
|
601
|
+
kvDtype: this.coldDtype,
|
|
602
|
+
counters: null,
|
|
601
603
|
};
|
|
602
604
|
return {
|
|
603
605
|
theoretical: hotStats.theoretical + coldStats.theoretical,
|
|
@@ -607,6 +609,11 @@ export class TieredKVCache {
|
|
|
607
609
|
seqLen: this.currentSeqLen,
|
|
608
610
|
maxSeqLen: this.maxSeqLen,
|
|
609
611
|
layout: this.layout,
|
|
612
|
+
kvDtype: this.kvDtype,
|
|
613
|
+
counters: {
|
|
614
|
+
hot: hotStats.counters ?? null,
|
|
615
|
+
cold: coldStats.counters ?? null,
|
|
616
|
+
},
|
|
610
617
|
hot: hotStats,
|
|
611
618
|
cold: coldStats,
|
|
612
619
|
};
|
|
@@ -183,6 +183,8 @@ export interface MemoryStats {
|
|
|
183
183
|
seqLen: number;
|
|
184
184
|
maxSeqLen: number;
|
|
185
185
|
layout: 'contiguous' | 'contiguous_quantized' | 'paged' | 'tiered' | 'bdpa_paged';
|
|
186
|
+
kvDtype?: 'f16' | 'f32' | null;
|
|
187
|
+
counters?: Record<string, unknown> | null;
|
|
186
188
|
}
|
|
187
189
|
|
|
188
190
|
/**
|