doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import { trace } from '../../../debug/index.js';
|
|
4
4
|
import { resolveCapturePolicy } from '../../../debug/capture-policy.js';
|
|
5
5
|
import { snapshotTensor, snapshotFromArray } from '../../../debug/tensor.js';
|
|
6
|
+
import { computeArrayStats } from '../../../debug/stats.js';
|
|
6
7
|
import { getDevice } from '../../../gpu/device.js';
|
|
7
8
|
import { allowReadback } from '../../../gpu/perf-guards.js';
|
|
8
9
|
import { f16ToF32 } from '../../../loader/dtype-utils.js';
|
|
@@ -11,10 +12,12 @@ import { PROBE_TO_CANONICAL } from './stage-names.js';
|
|
|
11
12
|
import { buildOpId } from './operator-identity.js';
|
|
12
13
|
import { getOperatorClass } from './stage-names.js';
|
|
13
14
|
import { getDriftPolicyId } from './drift-policy.js';
|
|
15
|
+
import { maybeWriteFixtureSnapshot } from './tsir-fixture-writer.js';
|
|
14
16
|
|
|
15
17
|
|
|
16
18
|
const STAGE_DEFAULT_CATEGORY = {
|
|
17
19
|
embed_out: 'embed',
|
|
20
|
+
per_layer_embed_out: 'embed',
|
|
18
21
|
// Attention stages (per-layer)
|
|
19
22
|
attn_input: 'attn',
|
|
20
23
|
post_input_norm: 'attn',
|
|
@@ -29,13 +32,17 @@ const STAGE_DEFAULT_CATEGORY = {
|
|
|
29
32
|
v_proj: 'attn',
|
|
30
33
|
q_norm: 'attn',
|
|
31
34
|
k_norm: 'attn',
|
|
35
|
+
v_norm: 'attn',
|
|
32
36
|
q_rope: 'attn',
|
|
33
37
|
k_rope: 'attn',
|
|
34
38
|
attn_scores: 'attn',
|
|
39
|
+
attn_core_out: 'attn',
|
|
35
40
|
attn_out: 'attn',
|
|
36
41
|
o_proj: 'attn',
|
|
37
42
|
post_attn: 'attn',
|
|
38
43
|
// FFN stages (per-layer)
|
|
44
|
+
per_layer_projection_in: 'ffn',
|
|
45
|
+
per_layer_projection_scaled: 'ffn',
|
|
39
46
|
per_layer_input: 'ffn',
|
|
40
47
|
per_layer_input_gate: 'ffn',
|
|
41
48
|
per_layer_input_activation: 'ffn',
|
|
@@ -107,17 +114,105 @@ function getTraceLogger(category, layerIdx) {
|
|
|
107
114
|
}
|
|
108
115
|
}
|
|
109
116
|
|
|
117
|
+
function formatProbeNumber(value) {
|
|
118
|
+
return Number.isFinite(value) ? value.toFixed(4) : String(value);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function alignUp4(value) {
|
|
122
|
+
return Math.ceil(value / 4) * 4;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
async function readGpuScalar(buffer, elementOffset, dtype, bytesPerElement) {
|
|
126
|
+
const byteOffset = elementOffset * bytesPerElement;
|
|
127
|
+
const alignedOffset = Math.floor(byteOffset / 4) * 4;
|
|
128
|
+
const offsetWithinRead = byteOffset - alignedOffset;
|
|
129
|
+
const readback = await readBufferSlice(buffer, alignedOffset, 4);
|
|
130
|
+
if (dtype === 'f16') {
|
|
131
|
+
const u16Array = new Uint16Array(readback);
|
|
132
|
+
return f16ToF32(u16Array[offsetWithinRead / 2]);
|
|
133
|
+
}
|
|
134
|
+
return new Float32Array(readback, offsetWithinRead, 1)[0];
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async function readTokenRow(buffer, tokenIdx, hiddenSize, dtype, isCpuBuffer) {
|
|
138
|
+
if (isCpuBuffer) {
|
|
139
|
+
const start = tokenIdx * hiddenSize;
|
|
140
|
+
const end = start + hiddenSize;
|
|
141
|
+
return buffer.slice(start, end);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const bytesPerElement = dtype === 'f16' ? 2 : 4;
|
|
145
|
+
const byteOffset = tokenIdx * hiddenSize * bytesPerElement;
|
|
146
|
+
const rowBytes = hiddenSize * bytesPerElement;
|
|
147
|
+
const alignedOffset = Math.floor(byteOffset / 4) * 4;
|
|
148
|
+
const offsetWithinRead = byteOffset - alignedOffset;
|
|
149
|
+
const readSize = alignUp4(offsetWithinRead + rowBytes);
|
|
150
|
+
const readback = await readBufferSlice(buffer, alignedOffset, readSize);
|
|
151
|
+
const values = new Float32Array(hiddenSize);
|
|
152
|
+
|
|
153
|
+
if (dtype === 'f16') {
|
|
154
|
+
const u16Array = new Uint16Array(readback);
|
|
155
|
+
const start = offsetWithinRead / 2;
|
|
156
|
+
for (let i = 0; i < hiddenSize; i++) {
|
|
157
|
+
values[i] = f16ToF32(u16Array[start + i]);
|
|
158
|
+
}
|
|
159
|
+
return values;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const f32Array = new Float32Array(readback, offsetWithinRead, hiddenSize);
|
|
163
|
+
values.set(f32Array);
|
|
164
|
+
return values;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function formatProbeStats(stats) {
|
|
168
|
+
return [
|
|
169
|
+
`min=${formatProbeNumber(stats.min)}`,
|
|
170
|
+
`max=${formatProbeNumber(stats.max)}`,
|
|
171
|
+
`mean=${formatProbeNumber(stats.mean)}`,
|
|
172
|
+
`std=${formatProbeNumber(stats.std)}`,
|
|
173
|
+
`maxAbs=${formatProbeNumber(stats.maxAbs)}`,
|
|
174
|
+
`valid=${stats.validCount}`,
|
|
175
|
+
`nan=${stats.nanCount}`,
|
|
176
|
+
`inf=${stats.infCount}`,
|
|
177
|
+
`zero=${stats.zeroCount}`,
|
|
178
|
+
].join(', ');
|
|
179
|
+
}
|
|
180
|
+
|
|
110
181
|
|
|
111
182
|
export async function runProbes(stage, buffer, options) {
|
|
112
183
|
const { layerIdx, numTokens, hiddenSize, probes, recorder, dtype = 'f32' } = options;
|
|
113
184
|
if (!buffer) return;
|
|
114
|
-
|
|
185
|
+
// Skip when a recorder is in flight unless operatorDiagnostics is
|
|
186
|
+
// actively capturing (diagnose mode emitter or tsirFixture writer).
|
|
187
|
+
if (recorder
|
|
188
|
+
&& !options.operatorDiagnostics?.enabled
|
|
189
|
+
&& !options.operatorDiagnostics?.tsirFixture?.dir) return;
|
|
115
190
|
|
|
116
191
|
const isCpuBuffer = buffer instanceof Float32Array;
|
|
117
192
|
const device = isCpuBuffer ? null : getDevice();
|
|
118
193
|
if (!isCpuBuffer && !device) return;
|
|
119
194
|
|
|
120
195
|
const diagnostics = options.operatorDiagnostics ?? null;
|
|
196
|
+
// TSIR fixture writer: when operatorDiagnostics.tsirFixture.dir is
|
|
197
|
+
// set and the stage maps to one of the four TSIR boundary points
|
|
198
|
+
// (post_input_norm/linear_qkv_proj/post_attn/layer_out), capture
|
|
199
|
+
// the activation tensor as a .npy file under
|
|
200
|
+
// <dir>/layer_<layerIdx>/<tsirStage>.npy. Builds the rung-5 frozen
|
|
201
|
+
// Doppler reference fixture that the Doe rung ladder consumes.
|
|
202
|
+
if (diagnostics?.tsirFixture?.dir) {
|
|
203
|
+
const tsirRecord = await maybeWriteFixtureSnapshot(stage, buffer, {
|
|
204
|
+
tsirFixture: diagnostics.tsirFixture,
|
|
205
|
+
layerIdx,
|
|
206
|
+
numTokens,
|
|
207
|
+
hiddenSize,
|
|
208
|
+
dtype,
|
|
209
|
+
recorder,
|
|
210
|
+
});
|
|
211
|
+
if (tsirRecord) {
|
|
212
|
+
const recordList = (diagnostics.tsirFixture.records ??= []);
|
|
213
|
+
recordList.push(tsirRecord);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
121
216
|
const canonicalStage = getCanonicalStageName(stage);
|
|
122
217
|
if (diagnostics?.enabled && diagnostics.emitter && canonicalStage) {
|
|
123
218
|
const opId = buildOpId(canonicalStage, layerIdx);
|
|
@@ -127,6 +222,7 @@ export async function runProbes(stage, buffer, options) {
|
|
|
127
222
|
isCpuBuffer,
|
|
128
223
|
shape: [numTokens, hiddenSize],
|
|
129
224
|
dtype,
|
|
225
|
+
recorder,
|
|
130
226
|
});
|
|
131
227
|
diagnostics.emitter.emitRecord(canonicalStage, {
|
|
132
228
|
layerIdx,
|
|
@@ -149,14 +245,14 @@ export async function runProbes(stage, buffer, options) {
|
|
|
149
245
|
if (stageProbes.length === 0) return;
|
|
150
246
|
if (!isCpuBuffer && !allowReadback(`probe.${stage}`)) return;
|
|
151
247
|
|
|
152
|
-
// Determine bytes per element based on dtype
|
|
153
248
|
const bytesPerElement = dtype === 'f16' ? 2 : 4;
|
|
154
249
|
|
|
155
250
|
for (const probe of stageProbes) {
|
|
156
251
|
if (!matchesLayer(probe.layers, layerIdx)) continue;
|
|
157
252
|
|
|
158
|
-
const dims = probe.dims
|
|
159
|
-
|
|
253
|
+
const dims = Array.isArray(probe.dims) ? probe.dims : [];
|
|
254
|
+
const includeStats = probe.stats === true;
|
|
255
|
+
if (dims.length === 0 && !includeStats) continue;
|
|
160
256
|
|
|
161
257
|
const tokens = resolveTokens(probe.tokens, numTokens);
|
|
162
258
|
if (tokens.length === 0) continue;
|
|
@@ -169,6 +265,12 @@ export async function runProbes(stage, buffer, options) {
|
|
|
169
265
|
|
|
170
266
|
for (const tokenIdx of tokens) {
|
|
171
267
|
|
|
268
|
+
let statsText = null;
|
|
269
|
+
if (includeStats) {
|
|
270
|
+
const row = await readTokenRow(buffer, tokenIdx, hiddenSize, dtype, isCpuBuffer);
|
|
271
|
+
statsText = formatProbeStats(computeArrayStats(row));
|
|
272
|
+
}
|
|
273
|
+
|
|
172
274
|
const values = [];
|
|
173
275
|
for (const dimIdx of dims) {
|
|
174
276
|
if (dimIdx < 0 || dimIdx >= hiddenSize) {
|
|
@@ -182,47 +284,42 @@ export async function runProbes(stage, buffer, options) {
|
|
|
182
284
|
continue;
|
|
183
285
|
}
|
|
184
286
|
const elementOffset = tokenIdx * hiddenSize + dimIdx;
|
|
185
|
-
const
|
|
186
|
-
|
|
187
|
-
const alignedOffset = Math.floor(byteOffset / 4) * 4;
|
|
188
|
-
const offsetWithinRead = byteOffset - alignedOffset;
|
|
189
|
-
const readSize = 4; // Always read 4 bytes (aligned)
|
|
190
|
-
const readback = await readBufferSlice(buffer, alignedOffset, readSize);
|
|
191
|
-
let value;
|
|
192
|
-
if (dtype === 'f16') {
|
|
193
|
-
// offsetWithinRead is 0 or 2 for F16 - extract correct u16
|
|
194
|
-
const u16Array = new Uint16Array(readback);
|
|
195
|
-
const u16Index = offsetWithinRead / 2;
|
|
196
|
-
value = f16ToF32(u16Array[u16Index]);
|
|
197
|
-
} else {
|
|
198
|
-
value = new Float32Array(readback)[0];
|
|
199
|
-
}
|
|
200
|
-
values.push(`${dimIdx}=${value.toFixed(4)}`);
|
|
287
|
+
const value = await readGpuScalar(buffer, elementOffset, dtype, bytesPerElement);
|
|
288
|
+
values.push(`${dimIdx}=${formatProbeNumber(value)}`);
|
|
201
289
|
}
|
|
202
290
|
|
|
203
|
-
|
|
291
|
+
const fields = [`PROBE${probeId} stage=${stage} token=${tokenIdx}`];
|
|
292
|
+
if (statsText) {
|
|
293
|
+
fields.push(`stats=[${statsText}]`);
|
|
294
|
+
}
|
|
295
|
+
if (values.length > 0) {
|
|
296
|
+
fields.push(`values=[${values.join(', ')}]`);
|
|
297
|
+
}
|
|
298
|
+
logger(fields.join(' '));
|
|
204
299
|
}
|
|
205
300
|
}
|
|
206
301
|
}
|
|
207
302
|
|
|
208
303
|
|
|
209
|
-
|
|
210
|
-
if (!probes || probes.length === 0) return false;
|
|
211
|
-
return probes.some((probe) => probe.stage === stage && matchesLayer(probe.layers, layerIdx));
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
export function getCanonicalStageName(probeStageName) {
|
|
304
|
+
function getCanonicalStageName(probeStageName) {
|
|
216
305
|
return PROBE_TO_CANONICAL[probeStageName] ?? null;
|
|
217
306
|
}
|
|
218
307
|
|
|
219
308
|
async function buildDiagnosticCapture(level, buffer, options) {
|
|
220
309
|
if (level === 'none') return null;
|
|
221
310
|
|
|
222
|
-
const { isCpuBuffer, shape, dtype } = options;
|
|
311
|
+
const { isCpuBuffer, shape, dtype, recorder } = options;
|
|
312
|
+
const includeData = level === 'full';
|
|
313
|
+
if (!isCpuBuffer && recorder) {
|
|
314
|
+
return createDeferredDiagnosticCapture(level, buffer, {
|
|
315
|
+
recorder,
|
|
316
|
+
shape,
|
|
317
|
+
dtype,
|
|
318
|
+
});
|
|
319
|
+
}
|
|
223
320
|
const snapshot = isCpuBuffer
|
|
224
|
-
? snapshotFromArray(buffer, shape, dtype)
|
|
225
|
-
: await snapshotTensor(buffer, shape, dtype);
|
|
321
|
+
? snapshotFromArray(buffer, shape, dtype, { includeData })
|
|
322
|
+
: await snapshotTensor(buffer, shape, dtype, { includeData });
|
|
226
323
|
if (!snapshot?.ok) {
|
|
227
324
|
return {
|
|
228
325
|
level,
|
|
@@ -242,5 +339,75 @@ async function buildDiagnosticCapture(level, buffer, options) {
|
|
|
242
339
|
stats: snapshot.stats ?? null,
|
|
243
340
|
hasNaN: snapshot.hasNaN === true,
|
|
244
341
|
hasInf: snapshot.hasInf === true,
|
|
342
|
+
data: Array.isArray(snapshot.data) ? snapshot.data : undefined,
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
function createDeferredDiagnosticCapture(level, buffer, options) {
|
|
347
|
+
const { recorder, shape, dtype } = options;
|
|
348
|
+
const elementSize = dtype === 'f16' ? 2 : 4;
|
|
349
|
+
const numElements = (shape ?? []).reduce((a, b) => a * b, 1);
|
|
350
|
+
const readSize = numElements > 0
|
|
351
|
+
? Math.min(buffer.size, numElements * elementSize)
|
|
352
|
+
: buffer.size;
|
|
353
|
+
const staging = recorder.device.createBuffer({
|
|
354
|
+
label: `${recorder.label}_diagnostic_capture`,
|
|
355
|
+
size: readSize,
|
|
356
|
+
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
|
357
|
+
});
|
|
358
|
+
recorder.getEncoder().copyBufferToBuffer(buffer, 0, staging, 0, readSize);
|
|
359
|
+
|
|
360
|
+
const capture = {
|
|
361
|
+
level,
|
|
362
|
+
shape,
|
|
363
|
+
dtype,
|
|
364
|
+
sample: null,
|
|
365
|
+
stats: null,
|
|
366
|
+
hasNaN: false,
|
|
367
|
+
hasInf: false,
|
|
245
368
|
};
|
|
369
|
+
|
|
370
|
+
recorder.enqueueCompletionTask(async () => {
|
|
371
|
+
let mapped = false;
|
|
372
|
+
try {
|
|
373
|
+
await staging.mapAsync(GPUMapMode.READ);
|
|
374
|
+
mapped = true;
|
|
375
|
+
const snapshot = snapshotFromArray(
|
|
376
|
+
decodeSnapshotBytes(staging.getMappedRange().slice(0), dtype),
|
|
377
|
+
shape,
|
|
378
|
+
dtype,
|
|
379
|
+
{ includeData: level === 'full' }
|
|
380
|
+
);
|
|
381
|
+
capture.shape = snapshot.shape;
|
|
382
|
+
capture.dtype = snapshot.dtype;
|
|
383
|
+
capture.sample = Array.isArray(snapshot.sample) ? snapshot.sample : null;
|
|
384
|
+
capture.stats = snapshot.stats ?? null;
|
|
385
|
+
capture.hasNaN = snapshot.hasNaN === true;
|
|
386
|
+
capture.hasInf = snapshot.hasInf === true;
|
|
387
|
+
capture.data = Array.isArray(snapshot.data) ? snapshot.data : undefined;
|
|
388
|
+
} catch (error) {
|
|
389
|
+
capture.error = error instanceof Error ? error.message : String(error);
|
|
390
|
+
capture.sample = null;
|
|
391
|
+
capture.stats = null;
|
|
392
|
+
} finally {
|
|
393
|
+
if (mapped) {
|
|
394
|
+
staging.unmap();
|
|
395
|
+
}
|
|
396
|
+
staging.destroy();
|
|
397
|
+
}
|
|
398
|
+
});
|
|
399
|
+
|
|
400
|
+
return capture;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
function decodeSnapshotBytes(data, dtype) {
|
|
404
|
+
if (dtype === 'f16') {
|
|
405
|
+
const src = new Uint16Array(data);
|
|
406
|
+
const decoded = new Float32Array(src.length);
|
|
407
|
+
for (let i = 0; i < src.length; i++) {
|
|
408
|
+
decoded[i] = f16ToF32(src[i]);
|
|
409
|
+
}
|
|
410
|
+
return decoded;
|
|
411
|
+
}
|
|
412
|
+
return new Float32Array(data);
|
|
246
413
|
}
|
|
@@ -82,6 +82,139 @@ function assertFiniteSamplingCandidates(logits, padTokenId, label) {
|
|
|
82
82
|
);
|
|
83
83
|
}
|
|
84
84
|
|
|
85
|
+
function isHigherPriorityCandidate(candidate, current) {
|
|
86
|
+
return candidate.logit > current.logit
|
|
87
|
+
|| (candidate.logit === current.logit && candidate.token < current.token);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function isLowerPriorityCandidate(candidate, current) {
|
|
91
|
+
return candidate.logit < current.logit
|
|
92
|
+
|| (candidate.logit === current.logit && candidate.token > current.token);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function heapSwap(heap, a, b) {
|
|
96
|
+
const tmp = heap[a];
|
|
97
|
+
heap[a] = heap[b];
|
|
98
|
+
heap[b] = tmp;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function heapPushLowestFirst(heap, candidate) {
|
|
102
|
+
heap.push(candidate);
|
|
103
|
+
let index = heap.length - 1;
|
|
104
|
+
while (index > 0) {
|
|
105
|
+
const parent = Math.floor((index - 1) / 2);
|
|
106
|
+
if (!isLowerPriorityCandidate(heap[index], heap[parent])) {
|
|
107
|
+
break;
|
|
108
|
+
}
|
|
109
|
+
heapSwap(heap, index, parent);
|
|
110
|
+
index = parent;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function heapifyLowestFirst(heap, index) {
|
|
115
|
+
while (true) {
|
|
116
|
+
const left = index * 2 + 1;
|
|
117
|
+
const right = left + 1;
|
|
118
|
+
let smallest = index;
|
|
119
|
+
if (left < heap.length && isLowerPriorityCandidate(heap[left], heap[smallest])) {
|
|
120
|
+
smallest = left;
|
|
121
|
+
}
|
|
122
|
+
if (right < heap.length && isLowerPriorityCandidate(heap[right], heap[smallest])) {
|
|
123
|
+
smallest = right;
|
|
124
|
+
}
|
|
125
|
+
if (smallest === index) {
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
heapSwap(heap, index, smallest);
|
|
129
|
+
index = smallest;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function selectTopKLogitCandidates(logits, topK) {
|
|
134
|
+
const limit = Math.max(1, Math.floor(topK));
|
|
135
|
+
if (!Number.isFinite(limit) || limit >= logits.length) {
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const heap = [];
|
|
140
|
+
for (let token = 0; token < logits.length; token++) {
|
|
141
|
+
const logit = logits[token];
|
|
142
|
+
if (!Number.isFinite(logit)) {
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
const candidate = { token, logit };
|
|
146
|
+
if (heap.length < limit) {
|
|
147
|
+
heapPushLowestFirst(heap, candidate);
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
if (isHigherPriorityCandidate(candidate, heap[0])) {
|
|
151
|
+
heap[0] = candidate;
|
|
152
|
+
heapifyLowestFirst(heap, 0);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
if (heap.length === 0) {
|
|
156
|
+
return [];
|
|
157
|
+
}
|
|
158
|
+
return heap.sort((a, b) => b.logit - a.logit || a.token - b.token);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function sampleFromLogitCandidates(candidates, temperature, seed, decode, debug, topK, topP) {
|
|
162
|
+
if (candidates.length === 0) {
|
|
163
|
+
throw new Error(
|
|
164
|
+
'[Sampling] Top-k filtering produced no finite candidates. ' +
|
|
165
|
+
'Upstream decode likely produced NaN/Inf logits.'
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
if (candidates.length === 1) {
|
|
169
|
+
return candidates[0].token;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const invTemperature = 1 / temperature;
|
|
173
|
+
let maxScaled = -Infinity;
|
|
174
|
+
for (const candidate of candidates) {
|
|
175
|
+
const scaled = candidate.logit * invTemperature;
|
|
176
|
+
candidate.scaled = scaled;
|
|
177
|
+
if (scaled > maxScaled) {
|
|
178
|
+
maxScaled = scaled;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
let sum = 0;
|
|
183
|
+
for (const candidate of candidates) {
|
|
184
|
+
const weight = Math.exp(candidate.scaled - maxScaled);
|
|
185
|
+
candidate.prob = weight;
|
|
186
|
+
sum += weight;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
if (sum > 0) {
|
|
190
|
+
const invSum = 1 / sum;
|
|
191
|
+
for (const candidate of candidates) {
|
|
192
|
+
candidate.prob *= invSum;
|
|
193
|
+
}
|
|
194
|
+
} else {
|
|
195
|
+
const uniformProb = 1.0 / candidates.length;
|
|
196
|
+
for (const candidate of candidates) {
|
|
197
|
+
candidate.prob = uniformProb;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
if (debug) {
|
|
202
|
+
const top5 = candidates.slice(0, 5).map(c => {
|
|
203
|
+
const text = decode?.([c.token]) ?? '?';
|
|
204
|
+
return `"${text}"(${(c.prob * 100).toFixed(1)}%)`;
|
|
205
|
+
});
|
|
206
|
+
trace.sample(`Top-5 (temp=${temperature}, topK=${topK}, topP=${topP}): ${top5.join(', ')}`);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
const r = seed !== undefined ? seededRandom(seed) : unseededRandom();
|
|
210
|
+
let cumProb = 0;
|
|
211
|
+
for (const candidate of candidates) {
|
|
212
|
+
cumProb += candidate.prob;
|
|
213
|
+
if (r < cumProb) return candidate.token;
|
|
214
|
+
}
|
|
215
|
+
return candidates[candidates.length - 1].token;
|
|
216
|
+
}
|
|
217
|
+
|
|
85
218
|
|
|
86
219
|
export function sample(logits, opts) {
|
|
87
220
|
const { temperature, topP, topK, decode, debug = false, padTokenId, seed } = opts;
|
|
@@ -119,6 +252,13 @@ export function sample(logits, opts) {
|
|
|
119
252
|
return maxIdx;
|
|
120
253
|
}
|
|
121
254
|
|
|
255
|
+
if (topP >= 1.0 && Number.isFinite(topK) && topK > 0) {
|
|
256
|
+
const candidates = selectTopKLogitCandidates(logits, topK);
|
|
257
|
+
if (candidates) {
|
|
258
|
+
return sampleFromLogitCandidates(candidates, temperature, seed, decode, debug, topK, topP);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
122
262
|
// Apply temperature
|
|
123
263
|
if (temperature !== 1.0) {
|
|
124
264
|
for (let i = 0; i < logits.length; i++) {
|
|
@@ -36,6 +36,7 @@ export const OPERATOR_CLASSES = Object.freeze({
|
|
|
36
36
|
export const STAGES = Object.freeze({
|
|
37
37
|
// Embedding
|
|
38
38
|
EMBED_OUT: 'embed.out',
|
|
39
|
+
PER_LAYER_EMBED_OUT: 'per_layer_embed.out',
|
|
39
40
|
|
|
40
41
|
// Attention stages (per-layer, prefixed with attn.)
|
|
41
42
|
ATTN_INPUT: 'attn.input',
|
|
@@ -51,6 +52,7 @@ export const STAGES = Object.freeze({
|
|
|
51
52
|
ATTN_K_ROPE: 'attn.k_rope',
|
|
52
53
|
ATTN_SCORES: 'attn.scores',
|
|
53
54
|
ATTN_SOFTMAX: 'attn.softmax',
|
|
55
|
+
ATTN_CORE_OUT: 'attn.core_out',
|
|
54
56
|
ATTN_OUT: 'attn.out',
|
|
55
57
|
ATTN_O_PROJ: 'attn.o_proj',
|
|
56
58
|
ATTN_POST_ATTN: 'attn.post_attn',
|
|
@@ -68,6 +70,8 @@ export const STAGES = Object.freeze({
|
|
|
68
70
|
CONV_OUT_PROJ: 'conv.out_proj',
|
|
69
71
|
|
|
70
72
|
// FFN stages (per-layer)
|
|
73
|
+
PER_LAYER_PROJECTION_IN: 'per_layer_projection.in',
|
|
74
|
+
PER_LAYER_PROJECTION_SCALED: 'per_layer_projection.scaled',
|
|
71
75
|
PER_LAYER_INPUT_OUT: 'per_layer_input.out',
|
|
72
76
|
FFN_NORMED: 'ffn.normed',
|
|
73
77
|
FFN_IN: 'ffn.in',
|
|
@@ -100,6 +104,7 @@ export const STAGES = Object.freeze({
|
|
|
100
104
|
|
|
101
105
|
const STAGE_TO_CLASS_MAP = Object.freeze({
|
|
102
106
|
[STAGES.EMBED_OUT]: OPERATOR_CLASSES.EMBEDDING,
|
|
107
|
+
[STAGES.PER_LAYER_EMBED_OUT]: OPERATOR_CLASSES.EMBEDDING,
|
|
103
108
|
|
|
104
109
|
[STAGES.ATTN_INPUT]: OPERATOR_CLASSES.NORMALIZATION,
|
|
105
110
|
[STAGES.ATTN_POST_INPUT_NORM]: OPERATOR_CLASSES.NORMALIZATION,
|
|
@@ -114,6 +119,7 @@ const STAGE_TO_CLASS_MAP = Object.freeze({
|
|
|
114
119
|
[STAGES.ATTN_K_ROPE]: OPERATOR_CLASSES.ROPE,
|
|
115
120
|
[STAGES.ATTN_SCORES]: OPERATOR_CLASSES.ATTENTION,
|
|
116
121
|
[STAGES.ATTN_SOFTMAX]: OPERATOR_CLASSES.ATTENTION,
|
|
122
|
+
[STAGES.ATTN_CORE_OUT]: OPERATOR_CLASSES.ATTENTION,
|
|
117
123
|
[STAGES.ATTN_OUT]: OPERATOR_CLASSES.ATTENTION,
|
|
118
124
|
[STAGES.ATTN_O_PROJ]: OPERATOR_CLASSES.PROJECTION,
|
|
119
125
|
[STAGES.ATTN_POST_ATTN]: OPERATOR_CLASSES.RESIDUAL,
|
|
@@ -128,6 +134,8 @@ const STAGE_TO_CLASS_MAP = Object.freeze({
|
|
|
128
134
|
[STAGES.CONV_KERNEL]: OPERATOR_CLASSES.CONV,
|
|
129
135
|
[STAGES.CONV_OUT_PROJ]: OPERATOR_CLASSES.CONV,
|
|
130
136
|
|
|
137
|
+
[STAGES.PER_LAYER_PROJECTION_IN]: OPERATOR_CLASSES.PROJECTION,
|
|
138
|
+
[STAGES.PER_LAYER_PROJECTION_SCALED]: OPERATOR_CLASSES.PROJECTION,
|
|
131
139
|
[STAGES.PER_LAYER_INPUT_OUT]: OPERATOR_CLASSES.RESIDUAL,
|
|
132
140
|
[STAGES.FFN_NORMED]: OPERATOR_CLASSES.NORMALIZATION,
|
|
133
141
|
[STAGES.FFN_IN]: OPERATOR_CLASSES.FFN,
|
|
@@ -155,6 +163,7 @@ const STAGE_TO_CLASS_MAP = Object.freeze({
|
|
|
155
163
|
|
|
156
164
|
export const PROBE_TO_CANONICAL = Object.freeze({
|
|
157
165
|
embed_out: STAGES.EMBED_OUT,
|
|
166
|
+
per_layer_embed_out: STAGES.PER_LAYER_EMBED_OUT,
|
|
158
167
|
attn_input: STAGES.ATTN_INPUT,
|
|
159
168
|
post_input_norm: STAGES.ATTN_POST_INPUT_NORM,
|
|
160
169
|
attn_normed: STAGES.ATTN_NORMED,
|
|
@@ -171,9 +180,12 @@ export const PROBE_TO_CANONICAL = Object.freeze({
|
|
|
171
180
|
q_rope: STAGES.ATTN_Q_ROPE,
|
|
172
181
|
k_rope: STAGES.ATTN_K_ROPE,
|
|
173
182
|
attn_scores: STAGES.ATTN_SCORES,
|
|
183
|
+
attn_core_out: STAGES.ATTN_CORE_OUT,
|
|
174
184
|
attn_out: STAGES.ATTN_OUT,
|
|
175
185
|
o_proj: STAGES.ATTN_O_PROJ,
|
|
176
186
|
post_attn: STAGES.ATTN_POST_ATTN,
|
|
187
|
+
per_layer_projection_in: STAGES.PER_LAYER_PROJECTION_IN,
|
|
188
|
+
per_layer_projection_scaled: STAGES.PER_LAYER_PROJECTION_SCALED,
|
|
177
189
|
per_layer_input: STAGES.PER_LAYER_INPUT_OUT,
|
|
178
190
|
ffn_normed: STAGES.FFN_NORMED,
|
|
179
191
|
ffn_in: STAGES.FFN_IN,
|